2024-11-08 18:39:36,837 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-08 18:39:36,854 main DEBUG Took 0.014811 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-08 18:39:36,855 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-08 18:39:36,855 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-08 18:39:36,856 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-08 18:39:36,858 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,868 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-08 18:39:36,888 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,890 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,891 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,891 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,892 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,893 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,894 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,894 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,895 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,896 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,897 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,898 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,899 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,900 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,900 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,901 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,901 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,902 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,903 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,903 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,904 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,905 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-08 18:39:36,905 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,906 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-08 18:39:36,908 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-08 18:39:36,910 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-08 18:39:36,913 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-08 18:39:36,914 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-08 18:39:36,916 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-08 18:39:36,916 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-08 18:39:36,928 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-08 18:39:36,932 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-08 18:39:36,934 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-08 18:39:36,935 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-08 18:39:36,935 main DEBUG createAppenders(={Console}) 2024-11-08 18:39:36,936 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-08 18:39:36,937 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-08 18:39:36,937 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-08 18:39:36,938 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-08 18:39:36,938 main DEBUG OutputStream closed 2024-11-08 18:39:36,939 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-08 18:39:36,939 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-08 18:39:36,939 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-08 18:39:37,049 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-08 18:39:37,053 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-08 18:39:37,055 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-08 18:39:37,056 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-08 18:39:37,057 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-08 18:39:37,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-08 18:39:37,058 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-08 18:39:37,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-08 18:39:37,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-08 18:39:37,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-08 18:39:37,060 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-08 18:39:37,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-08 18:39:37,061 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-08 18:39:37,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-08 18:39:37,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-08 18:39:37,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-08 18:39:37,063 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-08 18:39:37,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-08 18:39:37,066 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08 18:39:37,066 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-08 18:39:37,067 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-08 18:39:37,068 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-08T18:39:37,405 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc 2024-11-08 18:39:37,410 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-08 18:39:37,410 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-08T18:39:37,423 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-08T18:39:37,469 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-08T18:39:37,494 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a, deleteOnExit=true 2024-11-08T18:39:37,494 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-08T18:39:37,496 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/test.cache.data in system properties and HBase conf 2024-11-08T18:39:37,497 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/hadoop.tmp.dir in system properties and HBase conf 2024-11-08T18:39:37,498 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/hadoop.log.dir in system properties and HBase conf 2024-11-08T18:39:37,499 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-08T18:39:37,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-08T18:39:37,500 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-08T18:39:37,625 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-08T18:39:37,765 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-08T18:39:37,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-08T18:39:37,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-08T18:39:37,778 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-08T18:39:37,779 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T18:39:37,780 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-08T18:39:37,782 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-08T18:39:37,789 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-08T18:39:37,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T18:39:37,796 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-08T18:39:37,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/nfs.dump.dir in system properties and HBase conf 2024-11-08T18:39:37,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/java.io.tmpdir in system properties and HBase conf 2024-11-08T18:39:37,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-08T18:39:37,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-08T18:39:37,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-08T18:39:39,073 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-08T18:39:39,152 INFO [Time-limited test {}] log.Log(170): Logging initialized @3264ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-08T18:39:39,237 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T18:39:39,307 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T18:39:39,329 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T18:39:39,330 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T18:39:39,331 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-08T18:39:39,346 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T18:39:39,350 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/hadoop.log.dir/,AVAILABLE} 2024-11-08T18:39:39,351 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T18:39:39,600 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/java.io.tmpdir/jetty-localhost-33721-hadoop-hdfs-3_4_1-tests_jar-_-any-17208615431199649346/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T18:39:39,607 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:33721} 2024-11-08T18:39:39,607 INFO [Time-limited test {}] server.Server(415): Started @3720ms 2024-11-08T18:39:40,135 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-08T18:39:40,144 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-08T18:39:40,147 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-08T18:39:40,147 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-08T18:39:40,148 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-08T18:39:40,149 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/hadoop.log.dir/,AVAILABLE} 2024-11-08T18:39:40,150 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-08T18:39:40,271 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/java.io.tmpdir/jetty-localhost-35179-hadoop-hdfs-3_4_1-tests_jar-_-any-5671919838795293695/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T18:39:40,272 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:35179} 2024-11-08T18:39:40,272 INFO [Time-limited test {}] server.Server(415): Started @4385ms 2024-11-08T18:39:40,328 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-08T18:39:41,477 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/data/data2/current/BP-656518607-172.17.0.2-1731091178482/current, will proceed with Du for space computation calculation, 2024-11-08T18:39:41,477 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/data/data1/current/BP-656518607-172.17.0.2-1731091178482/current, will proceed with Du for space computation calculation, 2024-11-08T18:39:41,506 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-08T18:39:41,556 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x501a47facd7d2837 with lease ID 0x27975cc3291286c5: Processing first storage report for DS-5f8f9064-7515-473f-9f0c-ac597c4e9119 from datanode DatanodeRegistration(127.0.0.1:46719, datanodeUuid=d9e2d767-d14a-4c65-9b4d-aa0a9bb7ed83, infoPort=36047, infoSecurePort=0, ipcPort=42155, storageInfo=lv=-57;cid=testClusterID;nsid=1874626912;c=1731091178482) 2024-11-08T18:39:41,557 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x501a47facd7d2837 with lease ID 0x27975cc3291286c5: from storage DS-5f8f9064-7515-473f-9f0c-ac597c4e9119 node DatanodeRegistration(127.0.0.1:46719, datanodeUuid=d9e2d767-d14a-4c65-9b4d-aa0a9bb7ed83, infoPort=36047, infoSecurePort=0, ipcPort=42155, storageInfo=lv=-57;cid=testClusterID;nsid=1874626912;c=1731091178482), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T18:39:41,557 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x501a47facd7d2837 with lease ID 0x27975cc3291286c5: Processing first storage report for DS-431c9a5d-1829-417b-8a3b-ca53e3e77074 from datanode DatanodeRegistration(127.0.0.1:46719, datanodeUuid=d9e2d767-d14a-4c65-9b4d-aa0a9bb7ed83, infoPort=36047, infoSecurePort=0, ipcPort=42155, storageInfo=lv=-57;cid=testClusterID;nsid=1874626912;c=1731091178482) 2024-11-08T18:39:41,557 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x501a47facd7d2837 with lease ID 0x27975cc3291286c5: from storage DS-431c9a5d-1829-417b-8a3b-ca53e3e77074 node DatanodeRegistration(127.0.0.1:46719, datanodeUuid=d9e2d767-d14a-4c65-9b4d-aa0a9bb7ed83, infoPort=36047, infoSecurePort=0, ipcPort=42155, storageInfo=lv=-57;cid=testClusterID;nsid=1874626912;c=1731091178482), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-08T18:39:41,641 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc 2024-11-08T18:39:41,735 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/zookeeper_0, clientPort=49215, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-08T18:39:41,749 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=49215 2024-11-08T18:39:41,763 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T18:39:41,767 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T18:39:42,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741825_1001 (size=7) 2024-11-08T18:39:42,479 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 with version=8 2024-11-08T18:39:42,480 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/hbase-staging 2024-11-08T18:39:42,572 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-08T18:39:42,820 INFO [Time-limited test {}] client.ConnectionUtils(128): master/b75fe890c219:0 server-side Connection retries=45 2024-11-08T18:39:42,833 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T18:39:42,833 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T18:39:42,839 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T18:39:42,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T18:39:42,840 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T18:39:42,973 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-08T18:39:43,027 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-08T18:39:43,035 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-08T18:39:43,039 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T18:39:43,064 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 998 (auto-detected) 2024-11-08T18:39:43,065 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-08T18:39:43,100 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42293 2024-11-08T18:39:43,127 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42293 connecting to ZooKeeper ensemble=127.0.0.1:49215 2024-11-08T18:39:43,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:422930x0, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T18:39:43,262 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42293-0x1011d08c3510000 connected 2024-11-08T18:39:43,358 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T18:39:43,361 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T18:39:43,372 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T18:39:43,376 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787, hbase.cluster.distributed=false 2024-11-08T18:39:43,405 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T18:39:43,413 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42293 2024-11-08T18:39:43,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42293 2024-11-08T18:39:43,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42293 2024-11-08T18:39:43,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42293 2024-11-08T18:39:43,417 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42293 2024-11-08T18:39:43,518 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/b75fe890c219:0 server-side Connection retries=45 2024-11-08T18:39:43,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T18:39:43,520 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-08T18:39:43,521 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-08T18:39:43,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-08T18:39:43,521 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-08T18:39:43,524 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-08T18:39:43,527 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-08T18:39:43,528 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44447 2024-11-08T18:39:43,530 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44447 connecting to ZooKeeper ensemble=127.0.0.1:49215 2024-11-08T18:39:43,531 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T18:39:43,534 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T18:39:43,573 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:444470x0, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-08T18:39:43,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:444470x0, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T18:39:43,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44447-0x1011d08c3510001 connected 2024-11-08T18:39:43,577 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-08T18:39:43,586 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-08T18:39:43,589 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-08T18:39:43,595 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-08T18:39:43,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44447 2024-11-08T18:39:43,597 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44447 2024-11-08T18:39:43,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44447 2024-11-08T18:39:43,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44447 2024-11-08T18:39:43,606 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44447 2024-11-08T18:39:43,622 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;b75fe890c219:42293 2024-11-08T18:39:43,623 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/b75fe890c219,42293,1731091182659 2024-11-08T18:39:43,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T18:39:43,640 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T18:39:43,641 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/b75fe890c219,42293,1731091182659 2024-11-08T18:39:43,673 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:43,673 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-08T18:39:43,673 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:43,675 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-08T18:39:43,676 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/b75fe890c219,42293,1731091182659 from backup master directory 2024-11-08T18:39:43,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/b75fe890c219,42293,1731091182659 2024-11-08T18:39:43,690 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T18:39:43,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-08T18:39:43,690 WARN [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T18:39:43,691 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=b75fe890c219,42293,1731091182659 2024-11-08T18:39:43,692 INFO [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-08T18:39:43,694 INFO [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-08T18:39:43,751 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/hbase.id] with ID: dd42e0e3-0cec-490b-ade7-f4fb956445d9 2024-11-08T18:39:43,751 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/.tmp/hbase.id 2024-11-08T18:39:43,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741826_1002 (size=42) 2024-11-08T18:39:44,165 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/.tmp/hbase.id]:[hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/hbase.id] 2024-11-08T18:39:44,222 INFO [master/b75fe890c219:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-08T18:39:44,228 INFO [master/b75fe890c219:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-08T18:39:44,249 INFO [master/b75fe890c219:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-08T18:39:44,307 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:44,307 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:44,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741827_1003 (size=196) 2024-11-08T18:39:44,341 INFO [master/b75fe890c219:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:39:44,343 INFO [master/b75fe890c219:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-08T18:39:44,356 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:44,360 INFO [master/b75fe890c219:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T18:39:44,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741828_1004 (size=1189) 2024-11-08T18:39:44,807 INFO [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store 2024-11-08T18:39:44,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741829_1005 (size=34) 2024-11-08T18:39:45,237 INFO [master/b75fe890c219:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-08T18:39:45,241 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:39:45,243 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T18:39:45,243 INFO [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T18:39:45,243 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T18:39:45,245 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T18:39:45,245 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T18:39:45,245 INFO [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T18:39:45,247 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731091185243Disabling compacts and flushes for region at 1731091185243Disabling writes for close at 1731091185245 (+2 ms)Writing region close event to WAL at 1731091185245Closed at 1731091185245 2024-11-08T18:39:45,249 WARN [master/b75fe890c219:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/.initializing 2024-11-08T18:39:45,249 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/WALs/b75fe890c219,42293,1731091182659 2024-11-08T18:39:45,259 INFO [master/b75fe890c219:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T18:39:45,274 INFO [master/b75fe890c219:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b75fe890c219%2C42293%2C1731091182659, suffix=, logDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/WALs/b75fe890c219,42293,1731091182659, archiveDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/oldWALs, maxLogs=10 2024-11-08T18:39:45,296 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/WALs/b75fe890c219,42293,1731091182659/b75fe890c219%2C42293%2C1731091182659.1731091185279, exclude list is [], retry=0 2024-11-08T18:39:45,315 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46719,DS-5f8f9064-7515-473f-9f0c-ac597c4e9119,DISK] 2024-11-08T18:39:45,317 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-08T18:39:45,352 INFO [master/b75fe890c219:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/WALs/b75fe890c219,42293,1731091182659/b75fe890c219%2C42293%2C1731091182659.1731091185279 2024-11-08T18:39:45,352 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36047:36047)] 2024-11-08T18:39:45,353 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:39:45,353 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:39:45,357 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,358 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,422 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-08T18:39:45,426 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:45,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,433 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-08T18:39:45,433 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,434 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:39:45,435 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,438 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-08T18:39:45,438 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,439 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:39:45,440 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,442 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-08T18:39:45,443 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,444 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:39:45,444 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,448 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,449 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,455 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,456 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,460 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-08T18:39:45,463 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-08T18:39:45,468 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T18:39:45,469 INFO [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64779229, jitterRate=-0.034714266657829285}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-08T18:39:45,478 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731091185371Initializing all the Stores at 1731091185373 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731091185374 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091185374Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091185375 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091185375Cleaning up temporary data from old regions at 1731091185456 (+81 ms)Region opened successfully at 1731091185478 (+22 ms) 2024-11-08T18:39:45,480 INFO [master/b75fe890c219:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-08T18:39:45,514 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5878f5d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b75fe890c219/172.17.0.2:0 2024-11-08T18:39:45,543 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-08T18:39:45,552 INFO [master/b75fe890c219:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-08T18:39:45,552 INFO [master/b75fe890c219:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-08T18:39:45,555 INFO [master/b75fe890c219:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-08T18:39:45,556 INFO [master/b75fe890c219:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-08T18:39:45,561 INFO [master/b75fe890c219:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-08T18:39:45,561 INFO [master/b75fe890c219:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-08T18:39:45,589 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-08T18:39:45,601 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-08T18:39:45,614 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-08T18:39:45,617 INFO [master/b75fe890c219:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-08T18:39:45,618 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-08T18:39:45,622 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-08T18:39:45,625 INFO [master/b75fe890c219:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-08T18:39:45,629 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-08T18:39:45,639 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-08T18:39:45,641 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-08T18:39:45,647 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-08T18:39:45,672 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-08T18:39:45,680 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-08T18:39:45,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T18:39:45,689 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-08T18:39:45,689 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:45,689 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:45,692 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=b75fe890c219,42293,1731091182659, sessionid=0x1011d08c3510000, setting cluster-up flag (Was=false) 2024-11-08T18:39:45,714 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:45,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:45,739 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-08T18:39:45,742 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b75fe890c219,42293,1731091182659 2024-11-08T18:39:45,764 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:45,764 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:45,789 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-08T18:39:45,791 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=b75fe890c219,42293,1731091182659 2024-11-08T18:39:45,797 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-08T18:39:45,813 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(746): ClusterId : dd42e0e3-0cec-490b-ade7-f4fb956445d9 2024-11-08T18:39:45,816 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-08T18:39:45,824 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-08T18:39:45,824 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-08T18:39:45,832 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-08T18:39:45,832 DEBUG [RS:0;b75fe890c219:44447 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cdd0e7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=b75fe890c219/172.17.0.2:0 2024-11-08T18:39:45,846 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;b75fe890c219:44447 2024-11-08T18:39:45,849 INFO [RS:0;b75fe890c219:44447 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-08T18:39:45,849 INFO [RS:0;b75fe890c219:44447 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-08T18:39:45,849 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-08T18:39:45,852 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(2659): reportForDuty to master=b75fe890c219,42293,1731091182659 with port=44447, startcode=1731091183487 2024-11-08T18:39:45,862 DEBUG [RS:0;b75fe890c219:44447 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-08T18:39:45,874 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-08T18:39:45,886 INFO [master/b75fe890c219:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-08T18:39:45,892 INFO [master/b75fe890c219:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-08T18:39:45,897 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49055, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-08T18:39:45,898 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: b75fe890c219,42293,1731091182659 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-08T18:39:45,903 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42293 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:45,907 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/b75fe890c219:0, corePoolSize=5, maxPoolSize=5 2024-11-08T18:39:45,907 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/b75fe890c219:0, corePoolSize=5, maxPoolSize=5 2024-11-08T18:39:45,908 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/b75fe890c219:0, corePoolSize=5, maxPoolSize=5 2024-11-08T18:39:45,908 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/b75fe890c219:0, corePoolSize=5, maxPoolSize=5 2024-11-08T18:39:45,908 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/b75fe890c219:0, corePoolSize=10, maxPoolSize=10 2024-11-08T18:39:45,909 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:45,909 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/b75fe890c219:0, corePoolSize=2, maxPoolSize=2 2024-11-08T18:39:45,909 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:45,910 INFO [master/b75fe890c219:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731091215910 2024-11-08T18:39:45,912 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-08T18:39:45,914 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-08T18:39:45,914 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T18:39:45,915 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-08T18:39:45,918 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-08T18:39:45,919 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-08T18:39:45,919 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-08T18:39:45,919 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-08T18:39:45,920 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,920 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-08T18:39:45,920 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:45,925 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-08T18:39:45,926 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-08T18:39:45,927 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-08T18:39:45,930 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-08T18:39:45,931 INFO [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-08T18:39:45,932 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-08T18:39:45,932 WARN [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-08T18:39:45,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741831_1007 (size=1321) 2024-11-08T18:39:45,933 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/b75fe890c219:0:becomeActiveMaster-HFileCleaner.large.0-1731091185932,5,FailOnTimeoutGroup] 2024-11-08T18:39:45,934 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/b75fe890c219:0:becomeActiveMaster-HFileCleaner.small.0-1731091185933,5,FailOnTimeoutGroup] 2024-11-08T18:39:45,934 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:45,934 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-08T18:39:45,934 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-08T18:39:45,935 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 2024-11-08T18:39:45,936 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:45,936 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:45,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741832_1008 (size=32) 2024-11-08T18:39:45,946 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:39:45,949 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T18:39:45,952 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T18:39:45,952 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:45,953 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T18:39:45,957 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T18:39:45,957 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:45,958 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T18:39:45,961 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T18:39:45,961 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,962 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:45,963 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T18:39:45,965 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T18:39:45,966 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:45,967 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:45,967 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T18:39:45,969 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740 2024-11-08T18:39:45,970 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740 2024-11-08T18:39:45,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T18:39:45,973 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T18:39:45,974 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:39:45,977 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T18:39:45,982 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T18:39:45,983 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61870194, jitterRate=-0.0780622661113739}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:39:45,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731091185946Initializing all the Stores at 1731091185948 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731091185948Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731091185949 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091185949Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731091185949Cleaning up temporary data from old regions at 1731091185973 (+24 ms)Region opened successfully at 1731091185987 (+14 ms) 2024-11-08T18:39:45,987 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T18:39:45,988 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T18:39:45,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T18:39:45,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T18:39:45,988 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T18:39:45,990 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T18:39:45,990 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731091185987Disabling compacts and flushes for region at 1731091185987Disabling writes for close at 1731091185988 (+1 ms)Writing region close event to WAL at 1731091185989 (+1 ms)Closed at 1731091185989 2024-11-08T18:39:45,993 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T18:39:45,993 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-08T18:39:45,999 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-08T18:39:46,006 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T18:39:46,009 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-08T18:39:46,034 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(2659): reportForDuty to master=b75fe890c219,42293,1731091182659 with port=44447, startcode=1731091183487 2024-11-08T18:39:46,037 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42293 {}] master.ServerManager(363): Checking decommissioned status of RegionServer b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,040 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42293 {}] master.ServerManager(517): Registering regionserver=b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,050 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 2024-11-08T18:39:46,050 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40013 2024-11-08T18:39:46,050 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-08T18:39:46,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T18:39:46,065 DEBUG [RS:0;b75fe890c219:44447 {}] zookeeper.ZKUtil(111): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,065 WARN [RS:0;b75fe890c219:44447 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-08T18:39:46,066 INFO [RS:0;b75fe890c219:44447 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T18:39:46,066 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/WALs/b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,073 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [b75fe890c219,44447,1731091183487] 2024-11-08T18:39:46,099 INFO [RS:0;b75fe890c219:44447 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-08T18:39:46,113 INFO [RS:0;b75fe890c219:44447 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-08T18:39:46,118 INFO [RS:0;b75fe890c219:44447 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-08T18:39:46,118 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,119 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-08T18:39:46,124 INFO [RS:0;b75fe890c219:44447 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-08T18:39:46,125 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,125 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,125 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,126 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,126 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,126 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,126 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/b75fe890c219:0, corePoolSize=2, maxPoolSize=2 2024-11-08T18:39:46,126 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,126 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,126 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,126 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,127 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,127 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/b75fe890c219:0, corePoolSize=1, maxPoolSize=1 2024-11-08T18:39:46,127 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/b75fe890c219:0, corePoolSize=3, maxPoolSize=3 2024-11-08T18:39:46,127 DEBUG [RS:0;b75fe890c219:44447 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0, corePoolSize=3, maxPoolSize=3 2024-11-08T18:39:46,128 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,128 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,128 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,128 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,128 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,129 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=b75fe890c219,44447,1731091183487-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T18:39:46,146 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-08T18:39:46,148 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=b75fe890c219,44447,1731091183487-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,148 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,148 INFO [RS:0;b75fe890c219:44447 {}] regionserver.Replication(171): b75fe890c219,44447,1731091183487 started 2024-11-08T18:39:46,160 WARN [b75fe890c219:42293 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-08T18:39:46,165 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:46,166 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1482): Serving as b75fe890c219,44447,1731091183487, RpcServer on b75fe890c219/172.17.0.2:44447, sessionid=0x1011d08c3510001 2024-11-08T18:39:46,167 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-08T18:39:46,167 DEBUG [RS:0;b75fe890c219:44447 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,167 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b75fe890c219,44447,1731091183487' 2024-11-08T18:39:46,167 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-08T18:39:46,168 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-08T18:39:46,169 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-08T18:39:46,169 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-08T18:39:46,169 DEBUG [RS:0;b75fe890c219:44447 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,169 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'b75fe890c219,44447,1731091183487' 2024-11-08T18:39:46,170 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-08T18:39:46,170 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-08T18:39:46,171 DEBUG [RS:0;b75fe890c219:44447 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-08T18:39:46,171 INFO [RS:0;b75fe890c219:44447 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-08T18:39:46,171 INFO [RS:0;b75fe890c219:44447 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-08T18:39:46,278 INFO [RS:0;b75fe890c219:44447 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-08T18:39:46,281 INFO [RS:0;b75fe890c219:44447 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b75fe890c219%2C44447%2C1731091183487, suffix=, logDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/WALs/b75fe890c219,44447,1731091183487, archiveDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/oldWALs, maxLogs=32 2024-11-08T18:39:46,298 DEBUG [RS:0;b75fe890c219:44447 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/WALs/b75fe890c219,44447,1731091183487/b75fe890c219%2C44447%2C1731091183487.1731091186284, exclude list is [], retry=0 2024-11-08T18:39:46,302 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46719,DS-5f8f9064-7515-473f-9f0c-ac597c4e9119,DISK] 2024-11-08T18:39:46,305 INFO [RS:0;b75fe890c219:44447 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/WALs/b75fe890c219,44447,1731091183487/b75fe890c219%2C44447%2C1731091183487.1731091186284 2024-11-08T18:39:46,306 DEBUG [RS:0;b75fe890c219:44447 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36047:36047)] 2024-11-08T18:39:46,414 DEBUG [b75fe890c219:42293 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-08T18:39:46,429 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,434 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b75fe890c219,44447,1731091183487, state=OPENING 2024-11-08T18:39:46,455 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-08T18:39:46,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:46,464 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:39:46,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T18:39:46,465 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T18:39:46,467 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-08T18:39:46,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:39:46,642 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-08T18:39:46,645 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58285, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-08T18:39:46,655 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-08T18:39:46,656 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-08T18:39:46,656 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-08T18:39:46,661 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=b75fe890c219%2C44447%2C1731091183487.meta, suffix=.meta, logDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/WALs/b75fe890c219,44447,1731091183487, archiveDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/oldWALs, maxLogs=32 2024-11-08T18:39:46,685 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/WALs/b75fe890c219,44447,1731091183487/b75fe890c219%2C44447%2C1731091183487.meta.1731091186664.meta, exclude list is [], retry=0 2024-11-08T18:39:46,690 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46719,DS-5f8f9064-7515-473f-9f0c-ac597c4e9119,DISK] 2024-11-08T18:39:46,694 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/WALs/b75fe890c219,44447,1731091183487/b75fe890c219%2C44447%2C1731091183487.meta.1731091186664.meta 2024-11-08T18:39:46,695 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36047:36047)] 2024-11-08T18:39:46,695 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:39:46,697 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-08T18:39:46,699 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-08T18:39:46,704 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-08T18:39:46,708 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-08T18:39:46,709 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:39:46,710 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-08T18:39:46,710 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-08T18:39:46,714 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-08T18:39:46,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-08T18:39:46,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:46,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:46,720 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-08T18:39:46,724 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-08T18:39:46,725 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:46,726 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:46,727 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-08T18:39:46,729 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-08T18:39:46,729 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:46,730 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:46,731 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-08T18:39:46,733 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-08T18:39:46,733 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:46,735 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-08T18:39:46,735 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-08T18:39:46,739 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740 2024-11-08T18:39:46,742 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740 2024-11-08T18:39:46,747 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-08T18:39:46,747 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-08T18:39:46,749 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:39:46,753 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-08T18:39:46,756 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61563527, jitterRate=-0.08263196051120758}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:39:46,757 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-08T18:39:46,759 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731091186711Writing region info on filesystem at 1731091186711Initializing all the Stores at 1731091186714 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731091186714Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731091186714Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091186714Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731091186714Cleaning up temporary data from old regions at 1731091186747 (+33 ms)Running coprocessor post-open hooks at 1731091186757 (+10 ms)Region opened successfully at 1731091186759 (+2 ms) 2024-11-08T18:39:46,769 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731091186635 2024-11-08T18:39:46,784 DEBUG [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-08T18:39:46,785 INFO [RS_OPEN_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-08T18:39:46,787 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,790 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as b75fe890c219,44447,1731091183487, state=OPEN 2024-11-08T18:39:46,838 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T18:39:46,838 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-08T18:39:46,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T18:39:46,838 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-08T18:39:46,839 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:46,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-08T18:39:46,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=b75fe890c219,44447,1731091183487 in 370 msec 2024-11-08T18:39:46,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-08T18:39:46,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 851 msec 2024-11-08T18:39:46,862 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-08T18:39:46,862 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-08T18:39:46,887 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:46,888 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:46,912 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:46,915 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43097, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:46,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1140 sec 2024-11-08T18:39:46,947 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731091186947, completionTime=-1 2024-11-08T18:39:46,951 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-08T18:39:46,951 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-08T18:39:47,018 INFO [master/b75fe890c219:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-08T18:39:47,018 INFO [master/b75fe890c219:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731091247018 2024-11-08T18:39:47,018 INFO [master/b75fe890c219:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731091307018 2024-11-08T18:39:47,019 INFO [master/b75fe890c219:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 67 msec 2024-11-08T18:39:47,021 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b75fe890c219,42293,1731091182659-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:47,021 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b75fe890c219,42293,1731091182659-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:47,021 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b75fe890c219,42293,1731091182659-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:47,023 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-b75fe890c219:42293, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:47,023 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:47,024 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:47,034 DEBUG [master/b75fe890c219:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-08T18:39:47,057 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.366sec 2024-11-08T18:39:47,058 INFO [master/b75fe890c219:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-08T18:39:47,059 INFO [master/b75fe890c219:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-08T18:39:47,060 INFO [master/b75fe890c219:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-08T18:39:47,061 INFO [master/b75fe890c219:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-08T18:39:47,061 INFO [master/b75fe890c219:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-08T18:39:47,061 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b75fe890c219,42293,1731091182659-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-08T18:39:47,062 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b75fe890c219,42293,1731091182659-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-08T18:39:47,069 DEBUG [master/b75fe890c219:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-08T18:39:47,070 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-08T18:39:47,071 INFO [master/b75fe890c219:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=b75fe890c219,42293,1731091182659-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-08T18:39:47,127 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bca31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:47,129 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-08T18:39:47,129 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-08T18:39:47,131 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:47,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:47,142 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:47,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6faf04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:47,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:47,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:47,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:47,153 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:47,156 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@512c5941, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:47,156 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:47,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:47,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36130, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:47,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=b75fe890c219,42293,1731091182659 2024-11-08T18:39:47,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=303, ProcessCount=11, AvailableMemoryMB=5010 2024-11-08T18:39:47,203 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T18:39:47,209 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is b75fe890c219,42293,1731091182659 2024-11-08T18:39:47,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@33f36750 2024-11-08T18:39:47,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T18:39:47,215 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T18:39:47,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:39:47,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:39:47,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T18:39:47,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T18:39:47,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-08T18:39:47,234 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:47,237 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T18:39:47,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T18:39:47,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741835_1011 (size=959) 2024-11-08T18:39:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T18:39:47,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T18:39:47,656 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 2024-11-08T18:39:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741836_1012 (size=53) 2024-11-08T18:39:47,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T18:39:48,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:39:48,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing b4a84f4a6ffc3bb08667e2c0f91ea282, disabling compactions & flushes 2024-11-08T18:39:48,071 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:48,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:48,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. after waiting 0 ms 2024-11-08T18:39:48,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:48,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:48,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for b4a84f4a6ffc3bb08667e2c0f91ea282: Waiting for close lock at 1731091188071Disabling compacts and flushes for region at 1731091188071Disabling writes for close at 1731091188072 (+1 ms)Writing region close event to WAL at 1731091188072Closed at 1731091188072 2024-11-08T18:39:48,074 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T18:39:48,079 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731091188074"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731091188074"}]},"ts":"1731091188074"} 2024-11-08T18:39:48,085 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T18:39:48,088 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T18:39:48,092 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091188088"}]},"ts":"1731091188088"} 2024-11-08T18:39:48,096 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T18:39:48,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, ASSIGN}] 2024-11-08T18:39:48,101 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, ASSIGN 2024-11-08T18:39:48,103 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, ASSIGN; state=OFFLINE, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=false 2024-11-08T18:39:48,254 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b4a84f4a6ffc3bb08667e2c0f91ea282, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:39:48,261 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, ASSIGN because future has completed 2024-11-08T18:39:48,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:39:48,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T18:39:48,424 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:48,424 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:39:48,424 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,424 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:39:48,425 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,425 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,427 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,429 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4a84f4a6ffc3bb08667e2c0f91ea282 columnFamilyName A 2024-11-08T18:39:48,429 DEBUG [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:48,431 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] regionserver.HStore(327): Store=b4a84f4a6ffc3bb08667e2c0f91ea282/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:39:48,431 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,434 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4a84f4a6ffc3bb08667e2c0f91ea282 columnFamilyName B 2024-11-08T18:39:48,434 DEBUG [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:48,435 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] regionserver.HStore(327): Store=b4a84f4a6ffc3bb08667e2c0f91ea282/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:39:48,435 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,438 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b4a84f4a6ffc3bb08667e2c0f91ea282 columnFamilyName C 2024-11-08T18:39:48,438 DEBUG [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:39:48,439 INFO [StoreOpener-b4a84f4a6ffc3bb08667e2c0f91ea282-1 {}] regionserver.HStore(327): Store=b4a84f4a6ffc3bb08667e2c0f91ea282/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:39:48,439 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,441 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,442 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,444 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,445 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,446 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:39:48,449 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,453 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T18:39:48,454 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened b4a84f4a6ffc3bb08667e2c0f91ea282; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66693213, jitterRate=-0.006193682551383972}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:39:48,454 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:48,455 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for b4a84f4a6ffc3bb08667e2c0f91ea282: Running coprocessor pre-open hook at 1731091188425Writing region info on filesystem at 1731091188425Initializing all the Stores at 1731091188426 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091188427 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091188427Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091188427Cleaning up temporary data from old regions at 1731091188445 (+18 ms)Running coprocessor post-open hooks at 1731091188454 (+9 ms)Region opened successfully at 1731091188455 (+1 ms) 2024-11-08T18:39:48,457 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., pid=6, masterSystemTime=1731091188417 2024-11-08T18:39:48,461 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:48,461 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:48,463 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b4a84f4a6ffc3bb08667e2c0f91ea282, regionState=OPEN, openSeqNum=2, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:39:48,467 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:39:48,476 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-08T18:39:48,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 in 207 msec 2024-11-08T18:39:48,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-08T18:39:48,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, ASSIGN in 378 msec 2024-11-08T18:39:48,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T18:39:48,487 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091188487"}]},"ts":"1731091188487"} 2024-11-08T18:39:48,491 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T18:39:48,493 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T18:39:48,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2700 sec 2024-11-08T18:39:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-08T18:39:49,392 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:39:49,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0b3b59, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,401 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,401 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,403 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5edd1052, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,404 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,407 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34fab187, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,410 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,412 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,414 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,416 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T18:39:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b75fe890c219,42293,1731091182659 2024-11-08T18:39:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@14016810 2024-11-08T18:39:49,420 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T18:39:49,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T18:39:49,426 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340b46e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,426 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,428 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,428 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b2da332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,429 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,431 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e3687d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,433 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6edf4fef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,433 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,433 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,434 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,434 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f5a6b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,437 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b383f3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21f4b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,440 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30bed14c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,443 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d1d7645, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,445 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52294ca5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,446 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27ff6f12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,448 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,449 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f810a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d7926de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,452 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4712ff10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,453 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,454 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,455 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@500e7ebc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,457 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b48425, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,457 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,457 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,458 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14983026, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,461 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c7a39e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4454ce2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,463 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,463 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,464 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,464 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@425be669, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,465 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,466 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,468 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ad430e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,469 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a9c1c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,469 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,469 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,470 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40d5a1a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,472 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@522fc2db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,474 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cadbd93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,474 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:39:49,474 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:39:49,475 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:39:49,475 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:39:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:39:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19dd9b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:39:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:39:49,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:39:49,477 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:39:49,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b52a890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:39:49,481 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,481 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,481 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,481 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,485 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,485 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,486 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,489 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,489 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:39:49,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36184, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,494 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-08T18:39:49,507 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,507 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,509 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,509 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:39:49,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,512 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T18:39:49,512 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:39:49,514 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:39:49,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,516 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:39:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:39:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:39:49,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:39:49,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:39:49,574 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:39:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:49,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:39:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T18:39:49,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T18:39:49,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:49,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:49,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:49,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:49,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:49,742 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/5a7ef95d346b49369c322b9f1667d165 is 50, key is test_row_0/A:col10/1731091189563/Put/seqid=0 2024-11-08T18:39:49,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741837_1013 (size=14341) 2024-11-08T18:39:49,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091249781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,810 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091249802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091249804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091249807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091249814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T18:39:49,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T18:39:49,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:49,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:49,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:49,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:49,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:49,922 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091249920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091249921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,924 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091249931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,936 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091249935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,939 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:49,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091249940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:49,943 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T18:39:50,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:50,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091250131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091250131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T18:39:50,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091250142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091250150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091250153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/5a7ef95d346b49369c322b9f1667d165 2024-11-08T18:39:50,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T18:39:50,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:50,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/54b5e16e71824902a4c2ed75b8caadb9 is 50, key is test_row_0/B:col10/1731091189563/Put/seqid=0 2024-11-08T18:39:50,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741838_1014 (size=12001) 2024-11-08T18:39:50,315 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/54b5e16e71824902a4c2ed75b8caadb9 2024-11-08T18:39:50,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T18:39:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:50,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:50,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:50,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/803ef8c2544e493f8267f28a01e8fe04 is 50, key is test_row_0/C:col10/1731091189563/Put/seqid=0 2024-11-08T18:39:50,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741839_1015 (size=12001) 2024-11-08T18:39:50,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/803ef8c2544e493f8267f28a01e8fe04 2024-11-08T18:39:50,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/5a7ef95d346b49369c322b9f1667d165 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5a7ef95d346b49369c322b9f1667d165 2024-11-08T18:39:50,433 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5a7ef95d346b49369c322b9f1667d165, entries=200, sequenceid=13, filesize=14.0 K 2024-11-08T18:39:50,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/54b5e16e71824902a4c2ed75b8caadb9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/54b5e16e71824902a4c2ed75b8caadb9 2024-11-08T18:39:50,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/54b5e16e71824902a4c2ed75b8caadb9, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T18:39:50,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/803ef8c2544e493f8267f28a01e8fe04 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/803ef8c2544e493f8267f28a01e8fe04 2024-11-08T18:39:50,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091250442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091250450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091250451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091250460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:50,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091250461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:50,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:50,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:50,487 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/803ef8c2544e493f8267f28a01e8fe04, entries=150, sequenceid=13, filesize=11.7 K 2024-11-08T18:39:50,495 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=154.31 KB/158010 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 915ms, sequenceid=13, compaction requested=false 2024-11-08T18:39:50,497 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T18:39:50,499 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:50,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-08T18:39:50,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:50,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T18:39:50,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/9ef789d322c74fdf8f8832b28caa1342 is 50, key is test_row_0/A:col10/1731091189797/Put/seqid=0 2024-11-08T18:39:50,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741840_1016 (size=12001) 2024-11-08T18:39:50,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T18:39:50,946 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/9ef789d322c74fdf8f8832b28caa1342 2024-11-08T18:39:50,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9aef3500845a4bf0ab27ea6c6eefbf43 is 50, key is test_row_0/B:col10/1731091189797/Put/seqid=0 2024-11-08T18:39:51,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741841_1017 (size=12001) 2024-11-08T18:39:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:51,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:51,007 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9aef3500845a4bf0ab27ea6c6eefbf43 2024-11-08T18:39:51,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091251014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091251016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,030 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091251018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091251021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,035 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091251028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,036 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/77b313e7e2824472bcbf8af8b630a22e is 50, key is test_row_0/C:col10/1731091189797/Put/seqid=0 2024-11-08T18:39:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741842_1018 (size=12001) 2024-11-08T18:39:51,071 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/77b313e7e2824472bcbf8af8b630a22e 2024-11-08T18:39:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/9ef789d322c74fdf8f8832b28caa1342 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9ef789d322c74fdf8f8832b28caa1342 2024-11-08T18:39:51,104 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9ef789d322c74fdf8f8832b28caa1342, entries=150, sequenceid=39, filesize=11.7 K 2024-11-08T18:39:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9aef3500845a4bf0ab27ea6c6eefbf43 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9aef3500845a4bf0ab27ea6c6eefbf43 2024-11-08T18:39:51,131 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9aef3500845a4bf0ab27ea6c6eefbf43, entries=150, sequenceid=39, filesize=11.7 K 2024-11-08T18:39:51,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/77b313e7e2824472bcbf8af8b630a22e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/77b313e7e2824472bcbf8af8b630a22e 2024-11-08T18:39:51,164 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/77b313e7e2824472bcbf8af8b630a22e, entries=150, sequenceid=39, filesize=11.7 K 2024-11-08T18:39:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091251140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091251141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,168 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,174 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 663ms, sequenceid=39, compaction requested=false 2024-11-08T18:39:51,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:51,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:51,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-08T18:39:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-08T18:39:51,181 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:39:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:51,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:39:51,186 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-08T18:39:51,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6640 sec 2024-11-08T18:39:51,192 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6980 sec 2024-11-08T18:39:51,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e4ac323819e840668e21abae5586d2a9 is 50, key is test_row_0/A:col10/1731091191173/Put/seqid=0 2024-11-08T18:39:51,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741843_1019 (size=12001) 2024-11-08T18:39:51,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091251307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,322 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091251310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091251319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091251381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091251380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091251431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,435 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091251432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,440 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091251440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e4ac323819e840668e21abae5586d2a9 2024-11-08T18:39:51,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091251642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,645 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091251650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,652 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-08T18:39:51,661 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:39:51,661 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:39:51,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091251660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-08T18:39:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T18:39:51,666 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:39:51,668 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:39:51,668 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:39:51,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a19d172f478f4544a3027962ee0d493b is 50, key is test_row_0/B:col10/1731091191173/Put/seqid=0 2024-11-08T18:39:51,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741844_1020 (size=12001) 2024-11-08T18:39:51,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a19d172f478f4544a3027962ee0d493b 2024-11-08T18:39:51,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091251691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:51,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091251700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:51,706 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:51,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/c3097cd4d6284b3cb1e0ea26aa2fce80 is 50, key is test_row_0/C:col10/1731091191173/Put/seqid=0 2024-11-08T18:39:51,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741845_1021 (size=12001) 2024-11-08T18:39:51,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/c3097cd4d6284b3cb1e0ea26aa2fce80 2024-11-08T18:39:51,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e4ac323819e840668e21abae5586d2a9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e4ac323819e840668e21abae5586d2a9 2024-11-08T18:39:51,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T18:39:51,774 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e4ac323819e840668e21abae5586d2a9, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T18:39:51,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a19d172f478f4544a3027962ee0d493b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a19d172f478f4544a3027962ee0d493b 2024-11-08T18:39:51,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a19d172f478f4544a3027962ee0d493b, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T18:39:51,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/c3097cd4d6284b3cb1e0ea26aa2fce80 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c3097cd4d6284b3cb1e0ea26aa2fce80 2024-11-08T18:39:51,820 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c3097cd4d6284b3cb1e0ea26aa2fce80, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T18:39:51,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 640ms, sequenceid=52, compaction requested=true 2024-11-08T18:39:51,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:51,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-08T18:39:51,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:51,825 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:39:51,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:39:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:39:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:51,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:39:51,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/6cc9f304068c4f72adee7fdad6c31674 is 50, key is test_row_0/A:col10/1731091191303/Put/seqid=0 2024-11-08T18:39:51,837 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:51,837 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:51,842 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:51,842 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:51,846 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:39:51,847 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:51,847 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/54b5e16e71824902a4c2ed75b8caadb9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9aef3500845a4bf0ab27ea6c6eefbf43, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a19d172f478f4544a3027962ee0d493b] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=35.2 K 2024-11-08T18:39:51,848 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:51,848 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:39:51,848 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:51,848 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5a7ef95d346b49369c322b9f1667d165, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9ef789d322c74fdf8f8832b28caa1342, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e4ac323819e840668e21abae5586d2a9] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=37.4 K 2024-11-08T18:39:51,849 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 54b5e16e71824902a4c2ed75b8caadb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731091189561 2024-11-08T18:39:51,850 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9aef3500845a4bf0ab27ea6c6eefbf43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731091189767 2024-11-08T18:39:51,850 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a7ef95d346b49369c322b9f1667d165, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731091189533 2024-11-08T18:39:51,851 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting a19d172f478f4544a3027962ee0d493b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091191168 2024-11-08T18:39:51,852 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ef789d322c74fdf8f8832b28caa1342, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731091189767 2024-11-08T18:39:51,854 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4ac323819e840668e21abae5586d2a9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091191168 2024-11-08T18:39:51,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741846_1022 (size=12001) 2024-11-08T18:39:51,868 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/6cc9f304068c4f72adee7fdad6c31674 2024-11-08T18:39:51,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/2d930470ca394eb4b437abccb5e5812e is 50, key is test_row_0/B:col10/1731091191303/Put/seqid=0 2024-11-08T18:39:51,913 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:51,914 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/2daf14c683824f628165f0d361d4e829 is 50, key is test_row_0/A:col10/1731091191173/Put/seqid=0 2024-11-08T18:39:51,915 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#11 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:51,915 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/cf9704ca2c76470589d8b4d1d481c576 is 50, key is test_row_0/B:col10/1731091191173/Put/seqid=0 2024-11-08T18:39:51,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741847_1023 (size=12001) 2024-11-08T18:39:51,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741849_1025 (size=12104) 2024-11-08T18:39:51,951 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/2daf14c683824f628165f0d361d4e829 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2daf14c683824f628165f0d361d4e829 2024-11-08T18:39:51,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:51,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:51,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741848_1024 (size=12104) 2024-11-08T18:39:51,979 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into 2daf14c683824f628165f0d361d4e829(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:51,979 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:51,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T18:39:51,984 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091191823; duration=0sec 2024-11-08T18:39:51,985 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:51,985 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:39:51,985 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:51,990 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:51,990 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:39:51,990 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:51,991 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/803ef8c2544e493f8267f28a01e8fe04, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/77b313e7e2824472bcbf8af8b630a22e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c3097cd4d6284b3cb1e0ea26aa2fce80] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=35.2 K 2024-11-08T18:39:51,993 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 803ef8c2544e493f8267f28a01e8fe04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731091189561 2024-11-08T18:39:51,995 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77b313e7e2824472bcbf8af8b630a22e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1731091189767 2024-11-08T18:39:51,996 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3097cd4d6284b3cb1e0ea26aa2fce80, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091191168 2024-11-08T18:39:52,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091252008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091252009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091252009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,018 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:52,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,019 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/78a5ef9cc1924238a80fd07870c62776 is 50, key is test_row_0/C:col10/1731091191173/Put/seqid=0 2024-11-08T18:39:52,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741850_1026 (size=12104) 2024-11-08T18:39:52,049 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/78a5ef9cc1924238a80fd07870c62776 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/78a5ef9cc1924238a80fd07870c62776 2024-11-08T18:39:52,063 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into 78a5ef9cc1924238a80fd07870c62776(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:52,064 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:52,064 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091191834; duration=0sec 2024-11-08T18:39:52,064 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:52,064 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:39:52,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091252120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091252122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091252130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091252209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,221 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T18:39:52,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091252220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T18:39:52,313 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-08T18:39:52,324 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/2d930470ca394eb4b437abccb5e5812e 2024-11-08T18:39:52,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091252330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091252331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091252340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/70ae62abc699413889c4ea877c3508a6 is 50, key is test_row_0/C:col10/1731091191303/Put/seqid=0 2024-11-08T18:39:52,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741851_1027 (size=12001) 2024-11-08T18:39:52,378 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/70ae62abc699413889c4ea877c3508a6 2024-11-08T18:39:52,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/6cc9f304068c4f72adee7fdad6c31674 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/6cc9f304068c4f72adee7fdad6c31674 2024-11-08T18:39:52,400 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/cf9704ca2c76470589d8b4d1d481c576 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/cf9704ca2c76470589d8b4d1d481c576 2024-11-08T18:39:52,426 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into cf9704ca2c76470589d8b4d1d481c576(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:52,426 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:52,429 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091191834; duration=0sec 2024-11-08T18:39:52,429 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:52,429 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:39:52,436 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/6cc9f304068c4f72adee7fdad6c31674, entries=150, sequenceid=75, filesize=11.7 K 2024-11-08T18:39:52,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/2d930470ca394eb4b437abccb5e5812e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/2d930470ca394eb4b437abccb5e5812e 2024-11-08T18:39:52,453 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/2d930470ca394eb4b437abccb5e5812e, entries=150, sequenceid=75, filesize=11.7 K 2024-11-08T18:39:52,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/70ae62abc699413889c4ea877c3508a6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/70ae62abc699413889c4ea877c3508a6 2024-11-08T18:39:52,469 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/70ae62abc699413889c4ea877c3508a6, entries=150, sequenceid=75, filesize=11.7 K 2024-11-08T18:39:52,471 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 645ms, sequenceid=75, compaction requested=false 2024-11-08T18:39:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:52,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-08T18:39:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-08T18:39:52,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-08T18:39:52,479 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 806 msec 2024-11-08T18:39:52,483 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 820 msec 2024-11-08T18:39:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:52,643 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:39:52,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:39:52,660 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/8be5ede66b2940d78fecb9725cbeebcc is 50, key is test_row_0/A:col10/1731091192006/Put/seqid=0 2024-11-08T18:39:52,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741852_1028 (size=14341) 2024-11-08T18:39:52,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/8be5ede66b2940d78fecb9725cbeebcc 2024-11-08T18:39:52,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091252727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091252729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091252735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/7b3ffbca2fbf4478aed0c0082882229a is 50, key is test_row_0/B:col10/1731091192006/Put/seqid=0 2024-11-08T18:39:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741853_1029 (size=12001) 2024-11-08T18:39:52,772 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/7b3ffbca2fbf4478aed0c0082882229a 2024-11-08T18:39:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-08T18:39:52,800 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:39:52,800 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:39:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-08T18:39:52,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/37d6f44be51d4401b0ce35f161b1927d is 50, key is test_row_0/C:col10/1731091192006/Put/seqid=0 2024-11-08T18:39:52,807 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:39:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T18:39:52,811 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:39:52,811 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:39:52,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741854_1030 (size=12001) 2024-11-08T18:39:52,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/37d6f44be51d4401b0ce35f161b1927d 2024-11-08T18:39:52,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091252850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,858 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091252852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:52,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091252856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:52,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:52,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/8be5ede66b2940d78fecb9725cbeebcc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/8be5ede66b2940d78fecb9725cbeebcc 2024-11-08T18:39:52,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/8be5ede66b2940d78fecb9725cbeebcc, entries=200, sequenceid=92, filesize=14.0 K 2024-11-08T18:39:52,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/7b3ffbca2fbf4478aed0c0082882229a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/7b3ffbca2fbf4478aed0c0082882229a 2024-11-08T18:39:52,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/7b3ffbca2fbf4478aed0c0082882229a, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T18:39:52,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/37d6f44be51d4401b0ce35f161b1927d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/37d6f44be51d4401b0ce35f161b1927d 2024-11-08T18:39:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T18:39:52,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/37d6f44be51d4401b0ce35f161b1927d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-08T18:39:52,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 284ms, sequenceid=92, compaction requested=true 2024-11-08T18:39:52,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:52,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:39:52,930 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:52,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:52,930 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:52,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:39:52,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:52,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:39:52,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:52,932 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:52,932 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:39:52,933 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:52,933 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2daf14c683824f628165f0d361d4e829, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/6cc9f304068c4f72adee7fdad6c31674, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/8be5ede66b2940d78fecb9725cbeebcc] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=37.5 K 2024-11-08T18:39:52,933 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:52,933 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:39:52,934 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:52,934 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/cf9704ca2c76470589d8b4d1d481c576, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/2d930470ca394eb4b437abccb5e5812e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/7b3ffbca2fbf4478aed0c0082882229a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=35.3 K 2024-11-08T18:39:52,934 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2daf14c683824f628165f0d361d4e829, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091191168 2024-11-08T18:39:52,935 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting cf9704ca2c76470589d8b4d1d481c576, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091191168 2024-11-08T18:39:52,936 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6cc9f304068c4f72adee7fdad6c31674, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731091191303 2024-11-08T18:39:52,936 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d930470ca394eb4b437abccb5e5812e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731091191303 2024-11-08T18:39:52,937 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8be5ede66b2940d78fecb9725cbeebcc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731091191992 2024-11-08T18:39:52,937 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b3ffbca2fbf4478aed0c0082882229a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731091191992 2024-11-08T18:39:52,958 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:52,960 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/80bdd3bae6144dedb16d92a5b0f3aa14 is 50, key is test_row_0/B:col10/1731091192006/Put/seqid=0 2024-11-08T18:39:52,963 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#19 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:52,964 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/31f81d43ca8c45429fe266ff69c816fc is 50, key is test_row_0/A:col10/1731091192006/Put/seqid=0 2024-11-08T18:39:52,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-08T18:39:52,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:52,967 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T18:39:52,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741855_1031 (size=12207) 2024-11-08T18:39:52,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/54e6c1f0f8864a08b805a2f4081aed78 is 50, key is test_row_0/A:col10/1731091192731/Put/seqid=0 2024-11-08T18:39:52,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741856_1032 (size=12207) 2024-11-08T18:39:53,008 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/80bdd3bae6144dedb16d92a5b0f3aa14 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/80bdd3bae6144dedb16d92a5b0f3aa14 2024-11-08T18:39:53,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-08T18:39:53,025 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-08T18:39:53,028 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T18:39:53,028 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-08T18:39:53,028 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-08T18:39:53,028 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-08T18:39:53,028 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into 80bdd3bae6144dedb16d92a5b0f3aa14(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:53,029 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:53,029 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091192930; duration=0sec 2024-11-08T18:39:53,029 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:53,029 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:39:53,029 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:53,030 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T18:39:53,030 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T18:39:53,032 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:53,032 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:39:53,032 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:53,032 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/78a5ef9cc1924238a80fd07870c62776, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/70ae62abc699413889c4ea877c3508a6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/37d6f44be51d4401b0ce35f161b1927d] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=35.3 K 2024-11-08T18:39:53,033 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 78a5ef9cc1924238a80fd07870c62776, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091191168 2024-11-08T18:39:53,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741857_1033 (size=12001) 2024-11-08T18:39:53,035 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 70ae62abc699413889c4ea877c3508a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731091191303 2024-11-08T18:39:53,036 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 37d6f44be51d4401b0ce35f161b1927d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731091191992 2024-11-08T18:39:53,042 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/54e6c1f0f8864a08b805a2f4081aed78 2024-11-08T18:39:53,065 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:53,066 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/791ceff19f7e4413b9deca73c0b4e893 is 50, key is test_row_0/C:col10/1731091192006/Put/seqid=0 2024-11-08T18:39:53,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9befce67a21e4cc78e395b175b5c1dc9 is 50, key is test_row_0/B:col10/1731091192731/Put/seqid=0 2024-11-08T18:39:53,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:53,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:53,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741858_1034 (size=12207) 2024-11-08T18:39:53,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091253121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,129 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T18:39:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091253120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091253124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741859_1035 (size=12001) 2024-11-08T18:39:53,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091253229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,244 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:39:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,245 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091253243, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091253246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091253246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,252 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:39:53,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,262 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091253249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,263 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,264 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,403 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/31f81d43ca8c45429fe266ff69c816fc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/31f81d43ca8c45429fe266ff69c816fc 2024-11-08T18:39:53,423 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into 31f81d43ca8c45429fe266ff69c816fc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:53,425 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:53,425 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091192930; duration=0sec 2024-11-08T18:39:53,425 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:53,425 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:39:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T18:39:53,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091253460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091253460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091253470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,532 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/791ceff19f7e4413b9deca73c0b4e893 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/791ceff19f7e4413b9deca73c0b4e893 2024-11-08T18:39:53,547 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into 791ceff19f7e4413b9deca73c0b4e893(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:53,547 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:53,547 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091192930; duration=0sec 2024-11-08T18:39:53,547 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:53,547 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:39:53,556 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9befce67a21e4cc78e395b175b5c1dc9 2024-11-08T18:39:53,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ef55d74db11d4c0fb432f53cfb7f8705 is 50, key is test_row_0/C:col10/1731091192731/Put/seqid=0 2024-11-08T18:39:53,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741860_1036 (size=12001) 2024-11-08T18:39:53,622 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ef55d74db11d4c0fb432f53cfb7f8705 2024-11-08T18:39:53,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/54e6c1f0f8864a08b805a2f4081aed78 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/54e6c1f0f8864a08b805a2f4081aed78 2024-11-08T18:39:53,647 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/54e6c1f0f8864a08b805a2f4081aed78, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T18:39:53,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9befce67a21e4cc78e395b175b5c1dc9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9befce67a21e4cc78e395b175b5c1dc9 2024-11-08T18:39:53,666 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9befce67a21e4cc78e395b175b5c1dc9, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T18:39:53,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ef55d74db11d4c0fb432f53cfb7f8705 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef55d74db11d4c0fb432f53cfb7f8705 2024-11-08T18:39:53,680 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef55d74db11d4c0fb432f53cfb7f8705, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T18:39:53,682 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 715ms, sequenceid=114, compaction requested=false 2024-11-08T18:39:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:53,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:53,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-08T18:39:53,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-08T18:39:53,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-08T18:39:53,692 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 875 msec 2024-11-08T18:39:53,697 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 893 msec 2024-11-08T18:39:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:53,786 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:39:53,786 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T18:39:53,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/62f69fecf08e48cd9694127a6e8e31ba is 50, key is test_row_0/A:col10/1731091193119/Put/seqid=0 2024-11-08T18:39:53,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741861_1037 (size=16781) 2024-11-08T18:39:53,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091253901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,912 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091253910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091253911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,916 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,917 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:53,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-08T18:39:53,951 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:39:53,951 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:39:53,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-08T18:39:53,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T18:39:53,955 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:39:53,957 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:39:53,957 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:39:54,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091254020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091254020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091254029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,033 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T18:39:54,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T18:39:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091254229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091254230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091254239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/62f69fecf08e48cd9694127a6e8e31ba 2024-11-08T18:39:54,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T18:39:54,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T18:39:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/372da6f1a72647f9bdaa8f284896c6ad is 50, key is test_row_0/B:col10/1731091193119/Put/seqid=0 2024-11-08T18:39:54,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741862_1038 (size=12051) 2024-11-08T18:39:54,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T18:39:54,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:54,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091254540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091254550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:54,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091254550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:54,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T18:39:54,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T18:39:54,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:54,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,586 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/372da6f1a72647f9bdaa8f284896c6ad 2024-11-08T18:39:54,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T18:39:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:54,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:54,762 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/b4196be86593414baaa28024c51685d7 is 50, key is test_row_0/C:col10/1731091193119/Put/seqid=0 2024-11-08T18:39:54,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741863_1039 (size=12051) 2024-11-08T18:39:54,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/b4196be86593414baaa28024c51685d7 2024-11-08T18:39:54,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/62f69fecf08e48cd9694127a6e8e31ba as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62f69fecf08e48cd9694127a6e8e31ba 2024-11-08T18:39:54,815 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62f69fecf08e48cd9694127a6e8e31ba, entries=250, sequenceid=133, filesize=16.4 K 2024-11-08T18:39:54,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/372da6f1a72647f9bdaa8f284896c6ad as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/372da6f1a72647f9bdaa8f284896c6ad 2024-11-08T18:39:54,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/372da6f1a72647f9bdaa8f284896c6ad, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T18:39:54,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/b4196be86593414baaa28024c51685d7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/b4196be86593414baaa28024c51685d7 2024-11-08T18:39:54,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/b4196be86593414baaa28024c51685d7, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T18:39:54,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1062ms, sequenceid=133, compaction requested=true 2024-11-08T18:39:54,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:54,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:39:54,849 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:54,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:54,849 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:54,852 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:54,852 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:54,852 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:39:54,852 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:39:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:39:54,853 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,853 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:39:54,853 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:54,853 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/80bdd3bae6144dedb16d92a5b0f3aa14, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9befce67a21e4cc78e395b175b5c1dc9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/372da6f1a72647f9bdaa8f284896c6ad] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=35.4 K 2024-11-08T18:39:54,853 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/31f81d43ca8c45429fe266ff69c816fc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/54e6c1f0f8864a08b805a2f4081aed78, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62f69fecf08e48cd9694127a6e8e31ba] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=40.0 K 2024-11-08T18:39:54,854 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 31f81d43ca8c45429fe266ff69c816fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731091191992 2024-11-08T18:39:54,855 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 80bdd3bae6144dedb16d92a5b0f3aa14, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731091191992 2024-11-08T18:39:54,855 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54e6c1f0f8864a08b805a2f4081aed78, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731091192718 2024-11-08T18:39:54,856 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9befce67a21e4cc78e395b175b5c1dc9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731091192718 2024-11-08T18:39:54,856 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62f69fecf08e48cd9694127a6e8e31ba, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091193099 2024-11-08T18:39:54,857 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 372da6f1a72647f9bdaa8f284896c6ad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091193110 2024-11-08T18:39:54,885 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:54,886 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/e4081383fd2b402dae898f6af8f3f40f is 50, key is test_row_0/B:col10/1731091193119/Put/seqid=0 2024-11-08T18:39:54,899 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-08T18:39:54,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:54,900 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#28 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:54,900 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T18:39:54,900 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/4863f7a93fd74b44b5123ddc6c91d6c5 is 50, key is test_row_0/A:col10/1731091193119/Put/seqid=0 2024-11-08T18:39:54,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/5973bfbb5b294721b4a47e447ff610ed is 50, key is test_row_0/A:col10/1731091193896/Put/seqid=0 2024-11-08T18:39:54,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741864_1040 (size=12359) 2024-11-08T18:39:54,972 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/e4081383fd2b402dae898f6af8f3f40f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/e4081383fd2b402dae898f6af8f3f40f 2024-11-08T18:39:54,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741865_1041 (size=12359) 2024-11-08T18:39:55,000 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into e4081383fd2b402dae898f6af8f3f40f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:55,000 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:55,000 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091194849; duration=0sec 2024-11-08T18:39:55,000 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:55,000 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:39:55,001 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:55,007 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:55,007 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:39:55,007 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:55,007 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/791ceff19f7e4413b9deca73c0b4e893, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef55d74db11d4c0fb432f53cfb7f8705, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/b4196be86593414baaa28024c51685d7] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=35.4 K 2024-11-08T18:39:55,015 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 791ceff19f7e4413b9deca73c0b4e893, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731091191992 2024-11-08T18:39:55,016 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting ef55d74db11d4c0fb432f53cfb7f8705, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731091192718 2024-11-08T18:39:55,017 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting b4196be86593414baaa28024c51685d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091193110 2024-11-08T18:39:55,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741866_1042 (size=12151) 2024-11-08T18:39:55,060 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:55,065 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:55,066 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/2c95c73c5776405d894457f7ca3f384d is 50, key is test_row_0/C:col10/1731091193119/Put/seqid=0 2024-11-08T18:39:55,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T18:39:55,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741867_1043 (size=12359) 2024-11-08T18:39:55,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091255146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091255150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091255152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091255260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091255263, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,267 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4251 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:39:55,268 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,269 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:39:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091255261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,271 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091255280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091255271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,424 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/5973bfbb5b294721b4a47e447ff610ed 2024-11-08T18:39:55,435 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/4863f7a93fd74b44b5123ddc6c91d6c5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4863f7a93fd74b44b5123ddc6c91d6c5 2024-11-08T18:39:55,446 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into 4863f7a93fd74b44b5123ddc6c91d6c5(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:55,446 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:55,446 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091194849; duration=0sec 2024-11-08T18:39:55,447 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:55,447 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:39:55,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/677f18f931704841bcf03aa2e618ce35 is 50, key is test_row_0/B:col10/1731091193896/Put/seqid=0 2024-11-08T18:39:55,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091255480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091255500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,501 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091255500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741868_1044 (size=12151) 2024-11-08T18:39:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,505 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/677f18f931704841bcf03aa2e618ce35 2024-11-08T18:39:55,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/92fe07b842f849cfbe5b65a7a7297c44 is 50, key is test_row_0/C:col10/1731091193896/Put/seqid=0 2024-11-08T18:39:55,548 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/2c95c73c5776405d894457f7ca3f384d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/2c95c73c5776405d894457f7ca3f384d 2024-11-08T18:39:55,563 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into 2c95c73c5776405d894457f7ca3f384d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:55,563 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:55,563 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091194853; duration=0sec 2024-11-08T18:39:55,564 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:55,564 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:39:55,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741869_1045 (size=12151) 2024-11-08T18:39:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091255789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091255810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:55,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091255810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:55,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:55,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:55,971 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/92fe07b842f849cfbe5b65a7a7297c44 2024-11-08T18:39:55,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/5973bfbb5b294721b4a47e447ff610ed as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5973bfbb5b294721b4a47e447ff610ed 2024-11-08T18:39:56,011 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5973bfbb5b294721b4a47e447ff610ed, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T18:39:56,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/677f18f931704841bcf03aa2e618ce35 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/677f18f931704841bcf03aa2e618ce35 2024-11-08T18:39:56,029 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/677f18f931704841bcf03aa2e618ce35, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T18:39:56,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/92fe07b842f849cfbe5b65a7a7297c44 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/92fe07b842f849cfbe5b65a7a7297c44 2024-11-08T18:39:56,066 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/92fe07b842f849cfbe5b65a7a7297c44, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T18:39:56,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1168ms, sequenceid=153, compaction requested=false 2024-11-08T18:39:56,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:56,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:56,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-08T18:39:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-08T18:39:56,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-08T18:39:56,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1140 sec 2024-11-08T18:39:56,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1260 sec 2024-11-08T18:39:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-08T18:39:56,110 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:39:56,111 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:39:56,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-08T18:39:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T18:39:56,115 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:39:56,117 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:39:56,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:39:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T18:39:56,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-08T18:39:56,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:56,275 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T18:39:56,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e9683b8c1bd54abbae545af08350ec5c is 50, key is test_row_0/A:col10/1731091195146/Put/seqid=0 2024-11-08T18:39:56,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:56,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:56,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741870_1046 (size=12151) 2024-11-08T18:39:56,337 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e9683b8c1bd54abbae545af08350ec5c 2024-11-08T18:39:56,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/f4f12d1a02434123889f96582a79e800 is 50, key is test_row_0/B:col10/1731091195146/Put/seqid=0 2024-11-08T18:39:56,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091256397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091256397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091256397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,405 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741871_1047 (size=12151) 2024-11-08T18:39:56,423 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/f4f12d1a02434123889f96582a79e800 2024-11-08T18:39:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T18:39:56,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/0d64d2dbb0124c7db91ceafb8d481f7f is 50, key is test_row_0/C:col10/1731091195146/Put/seqid=0 2024-11-08T18:39:56,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741872_1048 (size=12151) 2024-11-08T18:39:56,463 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/0d64d2dbb0124c7db91ceafb8d481f7f 2024-11-08T18:39:56,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e9683b8c1bd54abbae545af08350ec5c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e9683b8c1bd54abbae545af08350ec5c 2024-11-08T18:39:56,496 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e9683b8c1bd54abbae545af08350ec5c, entries=150, sequenceid=172, filesize=11.9 K 2024-11-08T18:39:56,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/f4f12d1a02434123889f96582a79e800 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f4f12d1a02434123889f96582a79e800 2024-11-08T18:39:56,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091256510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,512 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f4f12d1a02434123889f96582a79e800, entries=150, sequenceid=172, filesize=11.9 K 2024-11-08T18:39:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091256510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/0d64d2dbb0124c7db91ceafb8d481f7f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/0d64d2dbb0124c7db91ceafb8d481f7f 2024-11-08T18:39:56,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091256511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,532 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/0d64d2dbb0124c7db91ceafb8d481f7f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-08T18:39:56,534 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 258ms, sequenceid=172, compaction requested=true 2024-11-08T18:39:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:56,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-08T18:39:56,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-08T18:39:56,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-08T18:39:56,540 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 419 msec 2024-11-08T18:39:56,543 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 430 msec 2024-11-08T18:39:56,723 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:39:56,723 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T18:39:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:56,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/05014fb754d54c84abfc914dede45e59 is 50, key is test_row_0/A:col10/1731091196722/Put/seqid=0 2024-11-08T18:39:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-08T18:39:56,741 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:39:56,741 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:39:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-08T18:39:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T18:39:56,747 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:39:56,749 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:39:56,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:39:56,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741873_1049 (size=14541) 2024-11-08T18:39:56,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091256767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091256769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,773 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091256772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T18:39:56,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091256879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091256879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:56,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091256880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:56,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:56,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T18:39:56,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:56,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:56,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:56,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:56,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:56,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T18:39:57,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:57,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,058 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T18:39:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091257089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091257090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:57,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:57,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091257094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,099 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:57,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/05014fb754d54c84abfc914dede45e59 2024-11-08T18:39:57,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/705018cc4972447e95a4f2dfe6a5143d is 50, key is test_row_0/B:col10/1731091196722/Put/seqid=0 2024-11-08T18:39:57,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741874_1050 (size=12151) 2024-11-08T18:39:57,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T18:39:57,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/705018cc4972447e95a4f2dfe6a5143d 2024-11-08T18:39:57,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:57,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/46c9cb6c11f44d1b8a38732fbbf571bd is 50, key is test_row_0/C:col10/1731091196722/Put/seqid=0 2024-11-08T18:39:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741875_1051 (size=12151) 2024-11-08T18:39:57,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T18:39:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:57,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T18:39:57,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:57,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091257399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:57,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:57,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091257401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:57,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:57,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091257415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:57,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:57,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T18:39:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T18:39:57,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:57,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:39:57,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/46c9cb6c11f44d1b8a38732fbbf571bd 2024-11-08T18:39:57,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/05014fb754d54c84abfc914dede45e59 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/05014fb754d54c84abfc914dede45e59 2024-11-08T18:39:57,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/05014fb754d54c84abfc914dede45e59, entries=200, sequenceid=194, filesize=14.2 K 2024-11-08T18:39:57,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/705018cc4972447e95a4f2dfe6a5143d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/705018cc4972447e95a4f2dfe6a5143d 2024-11-08T18:39:57,741 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/705018cc4972447e95a4f2dfe6a5143d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T18:39:57,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/46c9cb6c11f44d1b8a38732fbbf571bd as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/46c9cb6c11f44d1b8a38732fbbf571bd 2024-11-08T18:39:57,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/46c9cb6c11f44d1b8a38732fbbf571bd, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T18:39:57,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1034ms, sequenceid=194, compaction requested=true 2024-11-08T18:39:57,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:57,757 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:39:57,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:39:57,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:57,758 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:39:57,761 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:39:57,761 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:39:57,761 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,761 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/e4081383fd2b402dae898f6af8f3f40f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/677f18f931704841bcf03aa2e618ce35, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f4f12d1a02434123889f96582a79e800, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/705018cc4972447e95a4f2dfe6a5143d] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=47.7 K 2024-11-08T18:39:57,761 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51202 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:39:57,762 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:39:57,762 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,762 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4863f7a93fd74b44b5123ddc6c91d6c5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5973bfbb5b294721b4a47e447ff610ed, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e9683b8c1bd54abbae545af08350ec5c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/05014fb754d54c84abfc914dede45e59] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=50.0 K 2024-11-08T18:39:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:39:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:39:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:57,763 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4863f7a93fd74b44b5123ddc6c91d6c5, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091193110 2024-11-08T18:39:57,763 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting e4081383fd2b402dae898f6af8f3f40f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091193110 2024-11-08T18:39:57,765 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5973bfbb5b294721b4a47e447ff610ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731091193896 2024-11-08T18:39:57,765 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 677f18f931704841bcf03aa2e618ce35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731091193896 2024-11-08T18:39:57,766 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f4f12d1a02434123889f96582a79e800, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731091195128 2024-11-08T18:39:57,766 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9683b8c1bd54abbae545af08350ec5c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731091195128 2024-11-08T18:39:57,767 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 705018cc4972447e95a4f2dfe6a5143d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091196391 2024-11-08T18:39:57,767 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05014fb754d54c84abfc914dede45e59, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091196378 2024-11-08T18:39:57,801 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#39 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:57,802 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/a894f16dd01a43f39f57301732b58774 is 50, key is test_row_0/A:col10/1731091196722/Put/seqid=0 2024-11-08T18:39:57,806 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:57,807 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/527bcdebe6b24404a74fd27d2652ca52 is 50, key is test_row_0/B:col10/1731091196722/Put/seqid=0 2024-11-08T18:39:57,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741876_1052 (size=12595) 2024-11-08T18:39:57,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741877_1053 (size=12595) 2024-11-08T18:39:57,825 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/a894f16dd01a43f39f57301732b58774 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a894f16dd01a43f39f57301732b58774 2024-11-08T18:39:57,829 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/527bcdebe6b24404a74fd27d2652ca52 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/527bcdebe6b24404a74fd27d2652ca52 2024-11-08T18:39:57,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-08T18:39:57,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,838 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:39:57,843 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into a894f16dd01a43f39f57301732b58774(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:57,843 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:57,843 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=12, startTime=1731091197757; duration=0sec 2024-11-08T18:39:57,843 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:57,843 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:39:57,843 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:39:57,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/adb0fad71bbb4a0cad967c247fb1078a is 50, key is test_row_0/A:col10/1731091196762/Put/seqid=0 2024-11-08T18:39:57,850 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:39:57,850 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:39:57,850 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:57,850 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into 527bcdebe6b24404a74fd27d2652ca52(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:57,850 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/2c95c73c5776405d894457f7ca3f384d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/92fe07b842f849cfbe5b65a7a7297c44, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/0d64d2dbb0124c7db91ceafb8d481f7f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/46c9cb6c11f44d1b8a38732fbbf571bd] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=47.7 K 2024-11-08T18:39:57,850 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:57,851 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=12, startTime=1731091197758; duration=0sec 2024-11-08T18:39:57,851 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:57,851 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:39:57,853 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c95c73c5776405d894457f7ca3f384d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091193110 2024-11-08T18:39:57,853 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92fe07b842f849cfbe5b65a7a7297c44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731091193896 2024-11-08T18:39:57,854 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d64d2dbb0124c7db91ceafb8d481f7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731091195128 2024-11-08T18:39:57,855 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46c9cb6c11f44d1b8a38732fbbf571bd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091196391 2024-11-08T18:39:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T18:39:57,894 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:57,895 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/d96249457cce46c9aa61868427643f9c is 50, key is test_row_0/C:col10/1731091196722/Put/seqid=0 2024-11-08T18:39:57,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741878_1054 (size=12151) 2024-11-08T18:39:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:57,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:39:57,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741879_1055 (size=12595) 2024-11-08T18:39:57,960 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/d96249457cce46c9aa61868427643f9c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/d96249457cce46c9aa61868427643f9c 2024-11-08T18:39:57,973 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into d96249457cce46c9aa61868427643f9c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:57,973 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:57,973 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=12, startTime=1731091197762; duration=0sec 2024-11-08T18:39:57,973 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:57,973 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:39:57,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:57,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091257993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:57,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091257997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091257999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091258110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091258110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091258110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,114 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,297 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/adb0fad71bbb4a0cad967c247fb1078a 2024-11-08T18:39:58,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/1efb1634bfbb49608b4fed409e7bb1e3 is 50, key is test_row_0/B:col10/1731091196762/Put/seqid=0 2024-11-08T18:39:58,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091258320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091258320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091258321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741880_1056 (size=12151) 2024-11-08T18:39:58,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091258629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,630 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091258629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,631 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:58,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091258630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:58,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:58,741 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/1efb1634bfbb49608b4fed409e7bb1e3 2024-11-08T18:39:58,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/131f7b54383c40d6bd529cad19f0882d is 50, key is test_row_0/C:col10/1731091196762/Put/seqid=0 2024-11-08T18:39:58,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741881_1057 (size=12151) 2024-11-08T18:39:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T18:39:59,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091259141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091259140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091259140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,151 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,196 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/131f7b54383c40d6bd529cad19f0882d 2024-11-08T18:39:59,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/adb0fad71bbb4a0cad967c247fb1078a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/adb0fad71bbb4a0cad967c247fb1078a 2024-11-08T18:39:59,218 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/adb0fad71bbb4a0cad967c247fb1078a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-08T18:39:59,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/1efb1634bfbb49608b4fed409e7bb1e3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1efb1634bfbb49608b4fed409e7bb1e3 2024-11-08T18:39:59,232 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1efb1634bfbb49608b4fed409e7bb1e3, entries=150, sequenceid=209, filesize=11.9 K 2024-11-08T18:39:59,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/131f7b54383c40d6bd529cad19f0882d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/131f7b54383c40d6bd529cad19f0882d 2024-11-08T18:39:59,243 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/131f7b54383c40d6bd529cad19f0882d, entries=150, sequenceid=209, filesize=11.9 K 2024-11-08T18:39:59,245 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1407ms, sequenceid=209, compaction requested=false 2024-11-08T18:39:59,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:59,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:59,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-08T18:39:59,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-08T18:39:59,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-08T18:39:59,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4980 sec 2024-11-08T18:39:59,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.5100 sec 2024-11-08T18:39:59,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:59,303 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:39:59,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:39:59,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/7cef6bb64a724f19b7afa79b13bdeb06 is 50, key is test_row_0/A:col10/1731091199301/Put/seqid=0 2024-11-08T18:39:59,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741882_1058 (size=14541) 2024-11-08T18:39:59,339 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/7cef6bb64a724f19b7afa79b13bdeb06 2024-11-08T18:39:59,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091259347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091259350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,359 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,360 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a86c4a40b8564f15b586e63f66164d95 is 50, key is test_row_0/B:col10/1731091199301/Put/seqid=0 2024-11-08T18:39:59,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741883_1059 (size=12151) 2024-11-08T18:39:59,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a86c4a40b8564f15b586e63f66164d95 2024-11-08T18:39:59,430 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/3e23e3b716e54d45b56d18c7ecea5508 is 50, key is test_row_0/C:col10/1731091199301/Put/seqid=0 2024-11-08T18:39:59,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741884_1060 (size=12151) 2024-11-08T18:39:59,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/3e23e3b716e54d45b56d18c7ecea5508 2024-11-08T18:39:59,458 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/7cef6bb64a724f19b7afa79b13bdeb06 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7cef6bb64a724f19b7afa79b13bdeb06 2024-11-08T18:39:59,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091259462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,466 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,472 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091259469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,473 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7cef6bb64a724f19b7afa79b13bdeb06, entries=200, sequenceid=234, filesize=14.2 K 2024-11-08T18:39:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a86c4a40b8564f15b586e63f66164d95 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a86c4a40b8564f15b586e63f66164d95 2024-11-08T18:39:59,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a86c4a40b8564f15b586e63f66164d95, entries=150, sequenceid=234, filesize=11.9 K 2024-11-08T18:39:59,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/3e23e3b716e54d45b56d18c7ecea5508 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3e23e3b716e54d45b56d18c7ecea5508 2024-11-08T18:39:59,499 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3e23e3b716e54d45b56d18c7ecea5508, entries=150, sequenceid=234, filesize=11.9 K 2024-11-08T18:39:59,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 197ms, sequenceid=234, compaction requested=true 2024-11-08T18:39:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:39:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:59,500 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:39:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:39:59,500 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:39:59,500 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:59,502 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:59,502 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:39:59,502 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:59,502 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/527bcdebe6b24404a74fd27d2652ca52, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1efb1634bfbb49608b4fed409e7bb1e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a86c4a40b8564f15b586e63f66164d95] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=36.0 K 2024-11-08T18:39:59,502 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:59,503 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:39:59,503 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:59,503 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a894f16dd01a43f39f57301732b58774, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/adb0fad71bbb4a0cad967c247fb1078a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7cef6bb64a724f19b7afa79b13bdeb06] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=38.4 K 2024-11-08T18:39:59,503 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 527bcdebe6b24404a74fd27d2652ca52, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091196391 2024-11-08T18:39:59,503 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a894f16dd01a43f39f57301732b58774, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091196391 2024-11-08T18:39:59,503 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1efb1634bfbb49608b4fed409e7bb1e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731091196762 2024-11-08T18:39:59,504 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting adb0fad71bbb4a0cad967c247fb1078a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731091196762 2024-11-08T18:39:59,504 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting a86c4a40b8564f15b586e63f66164d95, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731091197990 2024-11-08T18:39:59,505 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cef6bb64a724f19b7afa79b13bdeb06, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731091197988 2024-11-08T18:39:59,517 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#48 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:59,518 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/eecf574f60a642e9a0c0d7496235e670 is 50, key is test_row_0/B:col10/1731091199301/Put/seqid=0 2024-11-08T18:39:59,537 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#49 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:39:59,538 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/a5cdf3a028194dbc8f974016af665ace is 50, key is test_row_0/A:col10/1731091199301/Put/seqid=0 2024-11-08T18:39:59,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741885_1061 (size=12697) 2024-11-08T18:39:59,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741886_1062 (size=12697) 2024-11-08T18:39:59,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:39:59,671 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:39:59,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:39:59,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/3de322453215432b84e4d5680ea8d87d is 50, key is test_row_0/A:col10/1731091199670/Put/seqid=0 2024-11-08T18:39:59,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741887_1063 (size=14541) 2024-11-08T18:39:59,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/3de322453215432b84e4d5680ea8d87d 2024-11-08T18:39:59,720 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9b089f69358942128aad825c095eab62 is 50, key is test_row_0/B:col10/1731091199670/Put/seqid=0 2024-11-08T18:39:59,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741888_1064 (size=12151) 2024-11-08T18:39:59,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091259741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091259745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091259849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,853 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:39:59,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091259849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:39:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:39:59,854 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:39:59,961 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/eecf574f60a642e9a0c0d7496235e670 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/eecf574f60a642e9a0c0d7496235e670 2024-11-08T18:39:59,984 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/a5cdf3a028194dbc8f974016af665ace as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a5cdf3a028194dbc8f974016af665ace 2024-11-08T18:39:59,987 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into eecf574f60a642e9a0c0d7496235e670(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:39:59,988 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:39:59,988 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091199500; duration=0sec 2024-11-08T18:39:59,988 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:39:59,988 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:39:59,988 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:39:59,989 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:39:59,990 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:39:59,990 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:39:59,990 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/d96249457cce46c9aa61868427643f9c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/131f7b54383c40d6bd529cad19f0882d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3e23e3b716e54d45b56d18c7ecea5508] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=36.0 K 2024-11-08T18:39:59,992 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d96249457cce46c9aa61868427643f9c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091196391 2024-11-08T18:39:59,993 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 131f7b54383c40d6bd529cad19f0882d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731091196762 2024-11-08T18:39:59,993 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e23e3b716e54d45b56d18c7ecea5508, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731091197990 2024-11-08T18:40:00,002 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into a5cdf3a028194dbc8f974016af665ace(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:00,002 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:00,002 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091199500; duration=0sec 2024-11-08T18:40:00,003 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:00,003 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:40:00,019 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:00,020 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/fed3b04206dd466d9207105e8f8923bb is 50, key is test_row_0/C:col10/1731091199301/Put/seqid=0 2024-11-08T18:40:00,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741889_1065 (size=12697) 2024-11-08T18:40:00,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091260061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091260060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,138 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9b089f69358942128aad825c095eab62 2024-11-08T18:40:00,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091260160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,162 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091260160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,163 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,164 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/18020bc61b194f678d641768ed9663f8 is 50, key is test_row_0/C:col10/1731091199670/Put/seqid=0 2024-11-08T18:40:00,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091260170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,180 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:00,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741890_1066 (size=12151) 2024-11-08T18:40:00,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091260369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,372 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091260380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,385 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,468 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/fed3b04206dd466d9207105e8f8923bb as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/fed3b04206dd466d9207105e8f8923bb 2024-11-08T18:40:00,476 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into fed3b04206dd466d9207105e8f8923bb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:00,476 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:00,476 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091199500; duration=0sec 2024-11-08T18:40:00,476 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:00,476 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:40:00,610 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/18020bc61b194f678d641768ed9663f8 2024-11-08T18:40:00,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/3de322453215432b84e4d5680ea8d87d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3de322453215432b84e4d5680ea8d87d 2024-11-08T18:40:00,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3de322453215432b84e4d5680ea8d87d, entries=200, sequenceid=248, filesize=14.2 K 2024-11-08T18:40:00,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9b089f69358942128aad825c095eab62 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9b089f69358942128aad825c095eab62 2024-11-08T18:40:00,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9b089f69358942128aad825c095eab62, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T18:40:00,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/18020bc61b194f678d641768ed9663f8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/18020bc61b194f678d641768ed9663f8 2024-11-08T18:40:00,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/18020bc61b194f678d641768ed9663f8, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T18:40:00,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 973ms, sequenceid=248, compaction requested=false 2024-11-08T18:40:00,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:00,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:00,884 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:00,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:40:00,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/2ed4c76904904c4291ba476c1b575d10 is 50, key is test_row_0/A:col10/1731091200882/Put/seqid=0 2024-11-08T18:40:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-08T18:40:00,900 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:00,901 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:00,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-08T18:40:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T18:40:00,904 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:00,905 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:00,905 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:00,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741891_1067 (size=14741) 2024-11-08T18:40:00,918 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/2ed4c76904904c4291ba476c1b575d10 2024-11-08T18:40:00,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091260919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,925 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091260924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:00,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:00,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/49587dd57479460baf316a83abcd9429 is 50, key is test_row_0/B:col10/1731091200882/Put/seqid=0 2024-11-08T18:40:00,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741892_1068 (size=12301) 2024-11-08T18:40:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T18:40:01,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:01,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091261029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,031 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:01,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:01,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091261040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,043 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:01,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T18:40:01,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:01,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T18:40:01,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:01,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T18:40:01,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:01,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091261239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091261250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:01,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/49587dd57479460baf316a83abcd9429 2024-11-08T18:40:01,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T18:40:01,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:01,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/8e275b5d52764e159f7cb41622f008e6 is 50, key is test_row_0/C:col10/1731091200882/Put/seqid=0 2024-11-08T18:40:01,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741893_1069 (size=12301) 2024-11-08T18:40:01,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T18:40:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:01,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,522 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T18:40:01,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091261549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:01,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:01,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091261560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:01,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:01,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T18:40:01,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:01,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,678 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T18:40:01,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:01,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:01,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/8e275b5d52764e159f7cb41622f008e6 2024-11-08T18:40:01,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/2ed4c76904904c4291ba476c1b575d10 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2ed4c76904904c4291ba476c1b575d10 2024-11-08T18:40:01,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2ed4c76904904c4291ba476c1b575d10, entries=200, sequenceid=274, filesize=14.4 K 2024-11-08T18:40:01,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/49587dd57479460baf316a83abcd9429 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/49587dd57479460baf316a83abcd9429 2024-11-08T18:40:01,871 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/49587dd57479460baf316a83abcd9429, entries=150, sequenceid=274, filesize=12.0 K 2024-11-08T18:40:01,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/8e275b5d52764e159f7cb41622f008e6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8e275b5d52764e159f7cb41622f008e6 2024-11-08T18:40:01,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8e275b5d52764e159f7cb41622f008e6, entries=150, sequenceid=274, filesize=12.0 K 2024-11-08T18:40:01,884 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1000ms, sequenceid=274, compaction requested=true 2024-11-08T18:40:01,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:01,884 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:01,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:01,885 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:01,885 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:01,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:01,886 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41979 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:01,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:01,886 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:01,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:01,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:40:01,887 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:40:01,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:01,887 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,887 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,887 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/eecf574f60a642e9a0c0d7496235e670, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9b089f69358942128aad825c095eab62, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/49587dd57479460baf316a83abcd9429] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=36.3 K 2024-11-08T18:40:01,887 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a5cdf3a028194dbc8f974016af665ace, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3de322453215432b84e4d5680ea8d87d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2ed4c76904904c4291ba476c1b575d10] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=41.0 K 2024-11-08T18:40:01,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting eecf574f60a642e9a0c0d7496235e670, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731091197990 2024-11-08T18:40:01,888 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5cdf3a028194dbc8f974016af665ace, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731091197990 2024-11-08T18:40:01,888 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de322453215432b84e4d5680ea8d87d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731091199326 2024-11-08T18:40:01,888 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b089f69358942128aad825c095eab62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731091199326 2024-11-08T18:40:01,889 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 49587dd57479460baf316a83abcd9429, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091199730 2024-11-08T18:40:01,889 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ed4c76904904c4291ba476c1b575d10, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091199730 2024-11-08T18:40:01,912 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:01,913 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/1b92075652ff4bb5802251fd5514f89c is 50, key is test_row_0/B:col10/1731091200882/Put/seqid=0 2024-11-08T18:40:01,927 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#58 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:01,928 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/4a9ab506ec814db38432194ff9c5ae92 is 50, key is test_row_0/A:col10/1731091200882/Put/seqid=0 2024-11-08T18:40:01,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741894_1070 (size=12949) 2024-11-08T18:40:01,965 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/1b92075652ff4bb5802251fd5514f89c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1b92075652ff4bb5802251fd5514f89c 2024-11-08T18:40:01,975 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into 1b92075652ff4bb5802251fd5514f89c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:01,975 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:01,975 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091201885; duration=0sec 2024-11-08T18:40:01,976 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:01,976 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:40:01,976 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:01,978 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:01,978 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:40:01,978 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,978 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/fed3b04206dd466d9207105e8f8923bb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/18020bc61b194f678d641768ed9663f8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8e275b5d52764e159f7cb41622f008e6] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=36.3 K 2024-11-08T18:40:01,979 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting fed3b04206dd466d9207105e8f8923bb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731091197990 2024-11-08T18:40:01,979 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 18020bc61b194f678d641768ed9663f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731091199326 2024-11-08T18:40:01,980 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e275b5d52764e159f7cb41622f008e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091199730 2024-11-08T18:40:01,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741895_1071 (size=12949) 2024-11-08T18:40:01,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-08T18:40:01,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:01,986 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:40:02,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/7937aff794a84f4d8fe083870bf220ae is 50, key is test_row_0/A:col10/1731091200921/Put/seqid=0 2024-11-08T18:40:02,001 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#59 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:02,002 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/8918dfac7d154cb68bf5a9a9f52ec50a is 50, key is test_row_0/C:col10/1731091200882/Put/seqid=0 2024-11-08T18:40:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T18:40:02,064 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:02,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741896_1072 (size=9857) 2024-11-08T18:40:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:02,065 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/7937aff794a84f4d8fe083870bf220ae 2024-11-08T18:40:02,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741897_1073 (size=12949) 2024-11-08T18:40:02,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/3ba4cca8a6de4f50998774cb975027a7 is 50, key is test_row_0/B:col10/1731091200921/Put/seqid=0 2024-11-08T18:40:02,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741898_1074 (size=9857) 2024-11-08T18:40:02,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091262179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,187 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091262182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,193 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091262186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,195 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091262190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,195 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091262199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,202 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,306 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091262300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,307 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091262301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,310 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,397 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/4a9ab506ec814db38432194ff9c5ae92 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4a9ab506ec814db38432194ff9c5ae92 2024-11-08T18:40:02,411 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into 4a9ab506ec814db38432194ff9c5ae92(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:02,411 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:02,411 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091201884; duration=0sec 2024-11-08T18:40:02,411 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:02,411 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:40:02,495 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/8918dfac7d154cb68bf5a9a9f52ec50a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8918dfac7d154cb68bf5a9a9f52ec50a 2024-11-08T18:40:02,506 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into 8918dfac7d154cb68bf5a9a9f52ec50a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:02,506 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:02,506 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091201887; duration=0sec 2024-11-08T18:40:02,506 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:02,506 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:40:02,521 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/3ba4cca8a6de4f50998774cb975027a7 2024-11-08T18:40:02,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091262521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091262520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/c321d58f930e4e30a11e27b2bda53ff6 is 50, key is test_row_0/C:col10/1731091200921/Put/seqid=0 2024-11-08T18:40:02,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741899_1075 (size=9857) 2024-11-08T18:40:02,589 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/c321d58f930e4e30a11e27b2bda53ff6 2024-11-08T18:40:02,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/7937aff794a84f4d8fe083870bf220ae as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7937aff794a84f4d8fe083870bf220ae 2024-11-08T18:40:02,613 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7937aff794a84f4d8fe083870bf220ae, entries=100, sequenceid=288, filesize=9.6 K 2024-11-08T18:40:02,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/3ba4cca8a6de4f50998774cb975027a7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ba4cca8a6de4f50998774cb975027a7 2024-11-08T18:40:02,629 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ba4cca8a6de4f50998774cb975027a7, entries=100, sequenceid=288, filesize=9.6 K 2024-11-08T18:40:02,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/c321d58f930e4e30a11e27b2bda53ff6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c321d58f930e4e30a11e27b2bda53ff6 2024-11-08T18:40:02,644 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c321d58f930e4e30a11e27b2bda53ff6, entries=100, sequenceid=288, filesize=9.6 K 2024-11-08T18:40:02,646 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 660ms, sequenceid=288, compaction requested=false 2024-11-08T18:40:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:02,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-08T18:40:02,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-08T18:40:02,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-08T18:40:02,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7430 sec 2024-11-08T18:40:02,655 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7520 sec 2024-11-08T18:40:02,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:02,833 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:02,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T18:40:02,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/3d9c6dee3062424e9905c97e9c3b8fa4 is 50, key is test_row_0/A:col10/1731091202179/Put/seqid=0 2024-11-08T18:40:02,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091262859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091262861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,865 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741900_1076 (size=12301) 2024-11-08T18:40:02,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091262970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,973 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:02,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:02,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091262970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:02,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:02,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-08T18:40:03,050 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:03,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:03,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-08T18:40:03,053 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:03,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T18:40:03,055 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:03,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:03,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T18:40:03,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:03,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091263179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:03,182 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:03,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:03,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091263179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:03,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:03,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T18:40:03,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:03,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/3d9c6dee3062424e9905c97e9c3b8fa4 2024-11-08T18:40:03,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a0ae632bc922422292ecbcce8d839e1c is 50, key is test_row_0/B:col10/1731091202179/Put/seqid=0 2024-11-08T18:40:03,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741901_1077 (size=12301) 2024-11-08T18:40:03,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T18:40:03,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T18:40:03,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:03,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:03,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091263490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:03,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:03,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:03,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:03,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:03,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091263490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:03,493 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:03,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T18:40:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,528 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T18:40:03,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T18:40:03,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:03,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a0ae632bc922422292ecbcce8d839e1c 2024-11-08T18:40:03,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ae1bb99f016c403e9c4b98eac3a0258f is 50, key is test_row_0/C:col10/1731091202179/Put/seqid=0 2024-11-08T18:40:03,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741902_1078 (size=12301) 2024-11-08T18:40:03,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T18:40:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:03,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:03,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:04,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T18:40:04,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:04,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091263999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091263999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:04,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:04,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:04,001 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:04,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:04,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:04,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:04,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:04,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=315 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ae1bb99f016c403e9c4b98eac3a0258f 2024-11-08T18:40:04,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T18:40:04,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:04,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:04,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:04,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/3d9c6dee3062424e9905c97e9c3b8fa4 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3d9c6dee3062424e9905c97e9c3b8fa4 2024-11-08T18:40:04,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3d9c6dee3062424e9905c97e9c3b8fa4, entries=150, sequenceid=315, filesize=12.0 K 2024-11-08T18:40:04,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/a0ae632bc922422292ecbcce8d839e1c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a0ae632bc922422292ecbcce8d839e1c 2024-11-08T18:40:04,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T18:40:04,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a0ae632bc922422292ecbcce8d839e1c, entries=150, sequenceid=315, filesize=12.0 K 2024-11-08T18:40:04,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ae1bb99f016c403e9c4b98eac3a0258f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ae1bb99f016c403e9c4b98eac3a0258f 2024-11-08T18:40:04,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ae1bb99f016c403e9c4b98eac3a0258f, entries=150, sequenceid=315, filesize=12.0 K 2024-11-08T18:40:04,233 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1400ms, sequenceid=315, compaction requested=true 2024-11-08T18:40:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:04,234 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:04,234 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:04,234 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:04,236 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:04,236 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:40:04,236 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,236 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4a9ab506ec814db38432194ff9c5ae92, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7937aff794a84f4d8fe083870bf220ae, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3d9c6dee3062424e9905c97e9c3b8fa4] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=34.3 K 2024-11-08T18:40:04,237 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:04,237 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:40:04,237 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,237 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1b92075652ff4bb5802251fd5514f89c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ba4cca8a6de4f50998774cb975027a7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a0ae632bc922422292ecbcce8d839e1c] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=34.3 K 2024-11-08T18:40:04,238 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a9ab506ec814db38432194ff9c5ae92, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091199730 2024-11-08T18:40:04,238 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b92075652ff4bb5802251fd5514f89c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091199730 2024-11-08T18:40:04,238 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7937aff794a84f4d8fe083870bf220ae, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731091200914 2024-11-08T18:40:04,239 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ba4cca8a6de4f50998774cb975027a7, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731091200914 2024-11-08T18:40:04,239 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9c6dee3062424e9905c97e9c3b8fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731091202100 2024-11-08T18:40:04,240 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting a0ae632bc922422292ecbcce8d839e1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731091202100 2024-11-08T18:40:04,255 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:04,255 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/161d35463453454da35424288204145b is 50, key is test_row_0/B:col10/1731091202179/Put/seqid=0 2024-11-08T18:40:04,259 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#67 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:04,260 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/369327c6200d4f4c8662970b7d7241cc is 50, key is test_row_0/A:col10/1731091202179/Put/seqid=0 2024-11-08T18:40:04,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741903_1079 (size=13051) 2024-11-08T18:40:04,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741904_1080 (size=13051) 2024-11-08T18:40:04,287 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/161d35463453454da35424288204145b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/161d35463453454da35424288204145b 2024-11-08T18:40:04,298 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into 161d35463453454da35424288204145b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:04,298 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:04,298 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091204234; duration=0sec 2024-11-08T18:40:04,299 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:04,299 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:40:04,299 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:04,304 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/369327c6200d4f4c8662970b7d7241cc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/369327c6200d4f4c8662970b7d7241cc 2024-11-08T18:40:04,305 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35107 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:04,305 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:40:04,305 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,305 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8918dfac7d154cb68bf5a9a9f52ec50a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c321d58f930e4e30a11e27b2bda53ff6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ae1bb99f016c403e9c4b98eac3a0258f] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=34.3 K 2024-11-08T18:40:04,307 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8918dfac7d154cb68bf5a9a9f52ec50a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091199730 2024-11-08T18:40:04,308 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting c321d58f930e4e30a11e27b2bda53ff6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731091200914 2024-11-08T18:40:04,309 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting ae1bb99f016c403e9c4b98eac3a0258f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731091202100 2024-11-08T18:40:04,315 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into 369327c6200d4f4c8662970b7d7241cc(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:04,316 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:04,316 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091204234; duration=0sec 2024-11-08T18:40:04,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-08T18:40:04,316 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:04,316 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:40:04,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,316 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:40:04,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/661d2b153f10402984baf531b929649a is 50, key is test_row_0/A:col10/1731091202838/Put/seqid=0 2024-11-08T18:40:04,345 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#69 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:04,345 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/17dd23c37d8f46f2841c9fca70dab3d0 is 50, key is test_row_0/C:col10/1731091202179/Put/seqid=0 2024-11-08T18:40:04,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741905_1081 (size=12301) 2024-11-08T18:40:04,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741906_1082 (size=13051) 2024-11-08T18:40:04,423 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/17dd23c37d8f46f2841c9fca70dab3d0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/17dd23c37d8f46f2841c9fca70dab3d0 2024-11-08T18:40:04,434 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into 17dd23c37d8f46f2841c9fca70dab3d0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:04,434 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:04,434 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091204234; duration=0sec 2024-11-08T18:40:04,434 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:04,434 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:40:04,792 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/661d2b153f10402984baf531b929649a 2024-11-08T18:40:04,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/203eb5c3c7984970b9c3b9d5032a332e is 50, key is test_row_0/B:col10/1731091202838/Put/seqid=0 2024-11-08T18:40:04,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741907_1083 (size=12301) 2024-11-08T18:40:04,814 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/203eb5c3c7984970b9c3b9d5032a332e 2024-11-08T18:40:04,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ef2fba2358c640cf8565b795f5d8022c is 50, key is test_row_0/C:col10/1731091202838/Put/seqid=0 2024-11-08T18:40:04,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741908_1084 (size=12301) 2024-11-08T18:40:04,854 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ef2fba2358c640cf8565b795f5d8022c 2024-11-08T18:40:04,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/661d2b153f10402984baf531b929649a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/661d2b153f10402984baf531b929649a 2024-11-08T18:40:04,876 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/661d2b153f10402984baf531b929649a, entries=150, sequenceid=328, filesize=12.0 K 2024-11-08T18:40:04,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/203eb5c3c7984970b9c3b9d5032a332e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/203eb5c3c7984970b9c3b9d5032a332e 2024-11-08T18:40:04,886 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/203eb5c3c7984970b9c3b9d5032a332e, entries=150, sequenceid=328, filesize=12.0 K 2024-11-08T18:40:04,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/ef2fba2358c640cf8565b795f5d8022c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef2fba2358c640cf8565b795f5d8022c 2024-11-08T18:40:04,897 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef2fba2358c640cf8565b795f5d8022c, entries=150, sequenceid=328, filesize=12.0 K 2024-11-08T18:40:04,899 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 582ms, sequenceid=328, compaction requested=false 2024-11-08T18:40:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:04,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-08T18:40:04,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-08T18:40:04,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-08T18:40:04,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8460 sec 2024-11-08T18:40:04,907 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.8540 sec 2024-11-08T18:40:05,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:05,029 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:05,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:40:05,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/08087200554c4476956bc7dc95cfeaad is 50, key is test_row_0/A:col10/1731091205012/Put/seqid=0 2024-11-08T18:40:05,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741909_1085 (size=17181) 2024-11-08T18:40:05,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091265093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,094 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091265093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-08T18:40:05,200 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:05,201 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-08T18:40:05,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T18:40:05,203 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:05,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,204 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:05,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091265200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:05,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091265200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T18:40:05,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-08T18:40:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:05,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:05,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:05,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:05,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091265409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091265409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,416 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/08087200554c4476956bc7dc95cfeaad 2024-11-08T18:40:05,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/3ad8038328c94d23956fda6ced03d0e8 is 50, key is test_row_0/B:col10/1731091205012/Put/seqid=0 2024-11-08T18:40:05,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741910_1086 (size=12301) 2024-11-08T18:40:05,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/3ad8038328c94d23956fda6ced03d0e8 2024-11-08T18:40:05,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-08T18:40:05,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:05,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:05,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:05,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:05,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:05,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T18:40:05,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/bfd7c0e091b34123bba867f56893007b is 50, key is test_row_0/C:col10/1731091205012/Put/seqid=0 2024-11-08T18:40:05,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741911_1087 (size=12301) 2024-11-08T18:40:05,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=341 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/bfd7c0e091b34123bba867f56893007b 2024-11-08T18:40:05,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/08087200554c4476956bc7dc95cfeaad as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/08087200554c4476956bc7dc95cfeaad 2024-11-08T18:40:05,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/08087200554c4476956bc7dc95cfeaad, entries=250, sequenceid=341, filesize=16.8 K 2024-11-08T18:40:05,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/3ad8038328c94d23956fda6ced03d0e8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ad8038328c94d23956fda6ced03d0e8 2024-11-08T18:40:05,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ad8038328c94d23956fda6ced03d0e8, entries=150, sequenceid=341, filesize=12.0 K 2024-11-08T18:40:05,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/bfd7c0e091b34123bba867f56893007b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bfd7c0e091b34123bba867f56893007b 2024-11-08T18:40:05,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bfd7c0e091b34123bba867f56893007b, entries=150, sequenceid=341, filesize=12.0 K 2024-11-08T18:40:05,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 577ms, sequenceid=341, compaction requested=true 2024-11-08T18:40:05,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:05,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:05,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:05,607 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:05,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:05,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:05,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:05,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T18:40:05,607 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:05,610 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:05,610 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:40:05,610 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:05,611 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/369327c6200d4f4c8662970b7d7241cc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/661d2b153f10402984baf531b929649a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/08087200554c4476956bc7dc95cfeaad] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=41.5 K 2024-11-08T18:40:05,612 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:05,612 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:40:05,612 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:05,612 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/161d35463453454da35424288204145b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/203eb5c3c7984970b9c3b9d5032a332e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ad8038328c94d23956fda6ced03d0e8] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=36.8 K 2024-11-08T18:40:05,613 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 369327c6200d4f4c8662970b7d7241cc, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731091202100 2024-11-08T18:40:05,613 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 161d35463453454da35424288204145b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731091202100 2024-11-08T18:40:05,616 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 203eb5c3c7984970b9c3b9d5032a332e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731091202838 2024-11-08T18:40:05,617 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 661d2b153f10402984baf531b929649a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731091202838 2024-11-08T18:40:05,617 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ad8038328c94d23956fda6ced03d0e8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731091205012 2024-11-08T18:40:05,618 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 08087200554c4476956bc7dc95cfeaad, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731091205012 2024-11-08T18:40:05,644 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:05,645 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/402e5f40242547e5b1f22471e39521cf is 50, key is test_row_0/B:col10/1731091205012/Put/seqid=0 2024-11-08T18:40:05,646 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#76 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:05,647 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/b63a73df0b6e46feb65b052e8e954ef7 is 50, key is test_row_0/A:col10/1731091205012/Put/seqid=0 2024-11-08T18:40:05,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-08T18:40:05,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:05,667 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:40:05,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741912_1088 (size=13153) 2024-11-08T18:40:05,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741913_1089 (size=13153) 2024-11-08T18:40:05,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e5894beb62224b5ea4b138effe03bc04 is 50, key is test_row_0/A:col10/1731091205092/Put/seqid=0 2024-11-08T18:40:05,682 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/402e5f40242547e5b1f22471e39521cf as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/402e5f40242547e5b1f22471e39521cf 2024-11-08T18:40:05,691 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into 402e5f40242547e5b1f22471e39521cf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:05,692 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:05,692 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091205607; duration=0sec 2024-11-08T18:40:05,692 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:05,692 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:40:05,692 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:05,694 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:05,694 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:40:05,694 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:05,694 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/17dd23c37d8f46f2841c9fca70dab3d0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef2fba2358c640cf8565b795f5d8022c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bfd7c0e091b34123bba867f56893007b] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=36.8 K 2024-11-08T18:40:05,695 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17dd23c37d8f46f2841c9fca70dab3d0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=315, earliestPutTs=1731091202100 2024-11-08T18:40:05,696 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef2fba2358c640cf8565b795f5d8022c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731091202838 2024-11-08T18:40:05,696 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfd7c0e091b34123bba867f56893007b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731091205012 2024-11-08T18:40:05,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741914_1090 (size=12301) 2024-11-08T18:40:05,707 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:05,707 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e5894beb62224b5ea4b138effe03bc04 2024-11-08T18:40:05,708 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/bb55c04e4a4a43109609d90939f4e9e0 is 50, key is test_row_0/C:col10/1731091205012/Put/seqid=0 2024-11-08T18:40:05,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:05,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:05,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/8c61b43c60df4441a954900333208d6e is 50, key is test_row_0/B:col10/1731091205092/Put/seqid=0 2024-11-08T18:40:05,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741915_1091 (size=13153) 2024-11-08T18:40:05,750 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/bb55c04e4a4a43109609d90939f4e9e0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bb55c04e4a4a43109609d90939f4e9e0 2024-11-08T18:40:05,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091265754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,758 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741916_1092 (size=12301) 2024-11-08T18:40:05,762 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/8c61b43c60df4441a954900333208d6e 2024-11-08T18:40:05,766 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into bb55c04e4a4a43109609d90939f4e9e0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:05,766 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:05,766 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091205607; duration=0sec 2024-11-08T18:40:05,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091265762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,767 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:05,767 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:40:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/4244ec6ddacd4b3185e708d0b07956a8 is 50, key is test_row_0/C:col10/1731091205092/Put/seqid=0 2024-11-08T18:40:05,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741917_1093 (size=12301) 2024-11-08T18:40:05,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T18:40:05,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091265870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:05,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091265879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091266079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,085 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/b63a73df0b6e46feb65b052e8e954ef7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/b63a73df0b6e46feb65b052e8e954ef7 2024-11-08T18:40:06,097 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into b63a73df0b6e46feb65b052e8e954ef7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:06,097 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:06,097 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091205607; duration=0sec 2024-11-08T18:40:06,097 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:06,097 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:40:06,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091266094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,205 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/4244ec6ddacd4b3185e708d0b07956a8 2024-11-08T18:40:06,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/e5894beb62224b5ea4b138effe03bc04 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e5894beb62224b5ea4b138effe03bc04 2024-11-08T18:40:06,224 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e5894beb62224b5ea4b138effe03bc04, entries=150, sequenceid=365, filesize=12.0 K 2024-11-08T18:40:06,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36200 deadline: 1731091266222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36226 deadline: 1731091266222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,227 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,228 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,228 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,229 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36178 deadline: 1731091266240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,246 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in b4a84f4a6ffc3bb08667e2c0f91ea282 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8249 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/8c61b43c60df4441a954900333208d6e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/8c61b43c60df4441a954900333208d6e 2024-11-08T18:40:06,281 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/8c61b43c60df4441a954900333208d6e, entries=150, sequenceid=365, filesize=12.0 K 2024-11-08T18:40:06,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/4244ec6ddacd4b3185e708d0b07956a8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4244ec6ddacd4b3185e708d0b07956a8 2024-11-08T18:40:06,292 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4244ec6ddacd4b3185e708d0b07956a8, entries=150, sequenceid=365, filesize=12.0 K 2024-11-08T18:40:06,293 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 626ms, sequenceid=365, compaction requested=false 2024-11-08T18:40:06,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:06,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-08T18:40:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-08T18:40:06,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-08T18:40:06,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0910 sec 2024-11-08T18:40:06,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.1030 sec 2024-11-08T18:40:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-08T18:40:06,340 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:06,340 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:06,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-08T18:40:06,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T18:40:06,344 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:06,345 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:06,345 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:06,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:06,393 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:06,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:40:06,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/cbbf042ec3c0428dbdb5afd506b1c9db is 50, key is test_row_0/A:col10/1731091205736/Put/seqid=0 2024-11-08T18:40:06,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741918_1094 (size=14741) 2024-11-08T18:40:06,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/cbbf042ec3c0428dbdb5afd506b1c9db 2024-11-08T18:40:06,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T18:40:06,475 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091266473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/24e591dddb6747498ca809711fe71c06 is 50, key is test_row_0/B:col10/1731091205736/Put/seqid=0 2024-11-08T18:40:06,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091266476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,483 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741919_1095 (size=12301) 2024-11-08T18:40:06,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091266579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091266589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:06,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:06,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T18:40:06,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091266790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,792 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:06,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091266800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:06,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:06,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:06,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:06,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/24e591dddb6747498ca809711fe71c06 2024-11-08T18:40:06,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/85801b7698364e06abe1b61c78a283fa is 50, key is test_row_0/C:col10/1731091205736/Put/seqid=0 2024-11-08T18:40:06,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:06,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:06,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:06,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T18:40:06,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741920_1096 (size=12301) 2024-11-08T18:40:06,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:06,980 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/85801b7698364e06abe1b61c78a283fa 2024-11-08T18:40:06,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/cbbf042ec3c0428dbdb5afd506b1c9db as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/cbbf042ec3c0428dbdb5afd506b1c9db 2024-11-08T18:40:06,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/cbbf042ec3c0428dbdb5afd506b1c9db, entries=200, sequenceid=381, filesize=14.4 K 2024-11-08T18:40:06,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/24e591dddb6747498ca809711fe71c06 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/24e591dddb6747498ca809711fe71c06 2024-11-08T18:40:07,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/24e591dddb6747498ca809711fe71c06, entries=150, sequenceid=381, filesize=12.0 K 2024-11-08T18:40:07,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/85801b7698364e06abe1b61c78a283fa as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/85801b7698364e06abe1b61c78a283fa 2024-11-08T18:40:07,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/85801b7698364e06abe1b61c78a283fa, entries=150, sequenceid=381, filesize=12.0 K 2024-11-08T18:40:07,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 629ms, sequenceid=381, compaction requested=true 2024-11-08T18:40:07,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:07,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:07,024 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:07,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:07,024 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:07,025 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:07,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:07,025 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:40:07,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:07,026 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:07,026 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/b63a73df0b6e46feb65b052e8e954ef7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e5894beb62224b5ea4b138effe03bc04, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/cbbf042ec3c0428dbdb5afd506b1c9db] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=39.3 K 2024-11-08T18:40:07,026 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:07,026 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:40:07,026 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,026 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/402e5f40242547e5b1f22471e39521cf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/8c61b43c60df4441a954900333208d6e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/24e591dddb6747498ca809711fe71c06] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=36.9 K 2024-11-08T18:40:07,027 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b63a73df0b6e46feb65b052e8e954ef7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731091205012 2024-11-08T18:40:07,027 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5894beb62224b5ea4b138effe03bc04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731091205081 2024-11-08T18:40:07,027 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 402e5f40242547e5b1f22471e39521cf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731091205012 2024-11-08T18:40:07,028 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbbf042ec3c0428dbdb5afd506b1c9db, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731091205736 2024-11-08T18:40:07,028 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c61b43c60df4441a954900333208d6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731091205081 2024-11-08T18:40:07,028 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 24e591dddb6747498ca809711fe71c06, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731091205736 2024-11-08T18:40:07,052 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#84 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:07,053 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/f6b5cd37e6ac42e595227406f93bd6b7 is 50, key is test_row_0/B:col10/1731091205736/Put/seqid=0 2024-11-08T18:40:07,054 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:07,055 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/4451ab5db86d4d548d859a7b93cdf921 is 50, key is test_row_0/A:col10/1731091205736/Put/seqid=0 2024-11-08T18:40:07,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741921_1097 (size=13255) 2024-11-08T18:40:07,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741922_1098 (size=13255) 2024-11-08T18:40:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:07,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:07,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:40:07,115 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/4451ab5db86d4d548d859a7b93cdf921 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4451ab5db86d4d548d859a7b93cdf921 2024-11-08T18:40:07,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/2f4ca6020df44586afbc9b76f3fd5729 is 50, key is test_row_0/A:col10/1731091207100/Put/seqid=0 2024-11-08T18:40:07,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:07,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:07,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,141 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into 4451ab5db86d4d548d859a7b93cdf921(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:07,141 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:07,141 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091207023; duration=0sec 2024-11-08T18:40:07,141 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:07,141 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:40:07,141 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:07,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741923_1099 (size=14741) 2024-11-08T18:40:07,143 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:07,143 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:40:07,144 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,144 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bb55c04e4a4a43109609d90939f4e9e0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4244ec6ddacd4b3185e708d0b07956a8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/85801b7698364e06abe1b61c78a283fa] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=36.9 K 2024-11-08T18:40:07,145 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb55c04e4a4a43109609d90939f4e9e0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=341, earliestPutTs=1731091205012 2024-11-08T18:40:07,145 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4244ec6ddacd4b3185e708d0b07956a8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731091205081 2024-11-08T18:40:07,146 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85801b7698364e06abe1b61c78a283fa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731091205736 2024-11-08T18:40:07,148 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/2f4ca6020df44586afbc9b76f3fd5729 2024-11-08T18:40:07,173 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:07,173 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/94f0162c256048ff87ce017aafd45e56 is 50, key is test_row_0/C:col10/1731091205736/Put/seqid=0 2024-11-08T18:40:07,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091267178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091267181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/75bb820df4da4376b077cd107577cb6e is 50, key is test_row_0/B:col10/1731091207100/Put/seqid=0 2024-11-08T18:40:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,189 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741925_1101 (size=12301) 2024-11-08T18:40:07,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/75bb820df4da4376b077cd107577cb6e 2024-11-08T18:40:07,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741924_1100 (size=13255) 2024-11-08T18:40:07,263 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/15c7d8ea0c6c4905bf7a178797faa5fc is 50, key is test_row_0/C:col10/1731091207100/Put/seqid=0 2024-11-08T18:40:07,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741926_1102 (size=12301) 2024-11-08T18:40:07,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091267300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091267300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,312 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:07,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T18:40:07,487 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/f6b5cd37e6ac42e595227406f93bd6b7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f6b5cd37e6ac42e595227406f93bd6b7 2024-11-08T18:40:07,499 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into f6b5cd37e6ac42e595227406f93bd6b7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:07,499 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:07,499 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091207024; duration=0sec 2024-11-08T18:40:07,499 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:07,499 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:40:07,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091267519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091267520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:07,661 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/94f0162c256048ff87ce017aafd45e56 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/94f0162c256048ff87ce017aafd45e56 2024-11-08T18:40:07,671 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into 94f0162c256048ff87ce017aafd45e56(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:07,672 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:07,672 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091207026; duration=0sec 2024-11-08T18:40:07,672 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:07,672 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:40:07,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=405 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/15c7d8ea0c6c4905bf7a178797faa5fc 2024-11-08T18:40:07,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/2f4ca6020df44586afbc9b76f3fd5729 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2f4ca6020df44586afbc9b76f3fd5729 2024-11-08T18:40:07,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2f4ca6020df44586afbc9b76f3fd5729, entries=200, sequenceid=405, filesize=14.4 K 2024-11-08T18:40:07,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/75bb820df4da4376b077cd107577cb6e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/75bb820df4da4376b077cd107577cb6e 2024-11-08T18:40:07,725 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/75bb820df4da4376b077cd107577cb6e, entries=150, sequenceid=405, filesize=12.0 K 2024-11-08T18:40:07,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/15c7d8ea0c6c4905bf7a178797faa5fc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/15c7d8ea0c6c4905bf7a178797faa5fc 2024-11-08T18:40:07,734 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/15c7d8ea0c6c4905bf7a178797faa5fc, entries=150, sequenceid=405, filesize=12.0 K 2024-11-08T18:40:07,735 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 633ms, sequenceid=405, compaction requested=false 2024-11-08T18:40:07,735 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:07,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-08T18:40:07,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:07,746 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:40:07,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/86c5b0ddf4cc4cf9840dd085cf1b1044 is 50, key is test_row_0/A:col10/1731091207176/Put/seqid=0 2024-11-08T18:40:07,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741927_1103 (size=12301) 2024-11-08T18:40:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:07,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. as already flushing 2024-11-08T18:40:07,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091267873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091267877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091267990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:07,991 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091267990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:07,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:08,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/86c5b0ddf4cc4cf9840dd085cf1b1044 2024-11-08T18:40:08,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9d3ccca73b1b49458e2f007d50f9f314 is 50, key is test_row_0/B:col10/1731091207176/Put/seqid=0 2024-11-08T18:40:08,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741928_1104 (size=12301) 2024-11-08T18:40:08,176 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9d3ccca73b1b49458e2f007d50f9f314 2024-11-08T18:40:08,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/93942675bb81481b832d25b6889ca562 is 50, key is test_row_0/C:col10/1731091207176/Put/seqid=0 2024-11-08T18:40:08,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741929_1105 (size=12301) 2024-11-08T18:40:08,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091268199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:08,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:08,201 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:08,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091268199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:08,202 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:08,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T18:40:08,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091268509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:08,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:08,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091268509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:08,594 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/93942675bb81481b832d25b6889ca562 2024-11-08T18:40:08,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/86c5b0ddf4cc4cf9840dd085cf1b1044 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/86c5b0ddf4cc4cf9840dd085cf1b1044 2024-11-08T18:40:08,610 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/86c5b0ddf4cc4cf9840dd085cf1b1044, entries=150, sequenceid=421, filesize=12.0 K 2024-11-08T18:40:08,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/9d3ccca73b1b49458e2f007d50f9f314 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9d3ccca73b1b49458e2f007d50f9f314 2024-11-08T18:40:08,620 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9d3ccca73b1b49458e2f007d50f9f314, entries=150, sequenceid=421, filesize=12.0 K 2024-11-08T18:40:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/93942675bb81481b832d25b6889ca562 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/93942675bb81481b832d25b6889ca562 2024-11-08T18:40:08,635 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/93942675bb81481b832d25b6889ca562, entries=150, sequenceid=421, filesize=12.0 K 2024-11-08T18:40:08,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 890ms, sequenceid=421, compaction requested=true 2024-11-08T18:40:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-08T18:40:08,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-08T18:40:08,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-08T18:40:08,643 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2960 sec 2024-11-08T18:40:08,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 2.3040 sec 2024-11-08T18:40:09,028 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:09,029 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:40:09,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:09,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/ac800afb61f24ae380bbb997a462a15f is 50, key is test_row_0/A:col10/1731091207871/Put/seqid=0 2024-11-08T18:40:09,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741930_1106 (size=14741) 2024-11-08T18:40:09,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/ac800afb61f24ae380bbb997a462a15f 2024-11-08T18:40:09,067 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091269064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091269065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/1081cfbf8f934fa7b053718455ee8e80 is 50, key is test_row_0/B:col10/1731091207871/Put/seqid=0 2024-11-08T18:40:09,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741931_1107 (size=12301) 2024-11-08T18:40:09,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/1081cfbf8f934fa7b053718455ee8e80 2024-11-08T18:40:09,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/831ec16f7986423085b1f58018e1b412 is 50, key is test_row_0/C:col10/1731091207871/Put/seqid=0 2024-11-08T18:40:09,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741932_1108 (size=12301) 2024-11-08T18:40:09,119 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=445 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/831ec16f7986423085b1f58018e1b412 2024-11-08T18:40:09,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/ac800afb61f24ae380bbb997a462a15f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/ac800afb61f24ae380bbb997a462a15f 2024-11-08T18:40:09,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/ac800afb61f24ae380bbb997a462a15f, entries=200, sequenceid=445, filesize=14.4 K 2024-11-08T18:40:09,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/1081cfbf8f934fa7b053718455ee8e80 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1081cfbf8f934fa7b053718455ee8e80 2024-11-08T18:40:09,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1081cfbf8f934fa7b053718455ee8e80, entries=150, sequenceid=445, filesize=12.0 K 2024-11-08T18:40:09,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/831ec16f7986423085b1f58018e1b412 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/831ec16f7986423085b1f58018e1b412 2024-11-08T18:40:09,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/831ec16f7986423085b1f58018e1b412, entries=150, sequenceid=445, filesize=12.0 K 2024-11-08T18:40:09,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 157ms, sequenceid=445, compaction requested=true 2024-11-08T18:40:09,185 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:09,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:09,185 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:09,185 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:09,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:09,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:09,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:09,186 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T18:40:09,186 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:09,187 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:09,187 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:40:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:09,188 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:09,188 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:40:09,188 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:09,188 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f6b5cd37e6ac42e595227406f93bd6b7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/75bb820df4da4376b077cd107577cb6e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9d3ccca73b1b49458e2f007d50f9f314, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1081cfbf8f934fa7b053718455ee8e80] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=49.0 K 2024-11-08T18:40:09,189 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55038 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:09,189 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:40:09,189 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:09,189 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4451ab5db86d4d548d859a7b93cdf921, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2f4ca6020df44586afbc9b76f3fd5729, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/86c5b0ddf4cc4cf9840dd085cf1b1044, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/ac800afb61f24ae380bbb997a462a15f] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=53.7 K 2024-11-08T18:40:09,190 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f6b5cd37e6ac42e595227406f93bd6b7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731091205736 2024-11-08T18:40:09,190 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4451ab5db86d4d548d859a7b93cdf921, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731091205736 2024-11-08T18:40:09,192 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 75bb820df4da4376b077cd107577cb6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731091206470 2024-11-08T18:40:09,192 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f4ca6020df44586afbc9b76f3fd5729, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731091206464 2024-11-08T18:40:09,193 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3ccca73b1b49458e2f007d50f9f314, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731091207154 2024-11-08T18:40:09,193 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86c5b0ddf4cc4cf9840dd085cf1b1044, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731091207154 2024-11-08T18:40:09,193 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1081cfbf8f934fa7b053718455ee8e80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731091207871 2024-11-08T18:40:09,194 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac800afb61f24ae380bbb997a462a15f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731091207871 2024-11-08T18:40:09,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/62dab5d912824fd497d9fd32cf659e05 is 50, key is test_row_0/A:col10/1731091209063/Put/seqid=0 2024-11-08T18:40:09,221 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:09,222 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/b559f960e314475793d0ee550489fe49 is 50, key is test_row_0/B:col10/1731091207871/Put/seqid=0 2024-11-08T18:40:09,245 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#98 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:09,246 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/70ecfcc2ff1341138ed6f518d3e35977 is 50, key is test_row_0/A:col10/1731091207871/Put/seqid=0 2024-11-08T18:40:09,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741933_1109 (size=14741) 2024-11-08T18:40:09,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/62dab5d912824fd497d9fd32cf659e05 2024-11-08T18:40:09,300 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091269294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,301 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091269301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741934_1110 (size=13391) 2024-11-08T18:40:09,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741935_1111 (size=13391) 2024-11-08T18:40:09,319 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/efec71725e4c480b85825711dfcdbf5d is 50, key is test_row_0/B:col10/1731091209063/Put/seqid=0 2024-11-08T18:40:09,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741936_1112 (size=12301) 2024-11-08T18:40:09,412 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091269411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,414 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091269412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,508 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-08T18:40:09,508 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:09,509 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-08T18:40:09,509 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:09,510 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-08T18:40:09,510 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:09,513 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:09,513 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:09,513 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:09,513 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:09,513 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:09,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:09,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:09,514 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:09,514 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:09,523 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-08T18:40:09,523 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:09,523 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:09,523 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:09,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:09,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091269626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091269626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,720 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/70ecfcc2ff1341138ed6f518d3e35977 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/70ecfcc2ff1341138ed6f518d3e35977 2024-11-08T18:40:09,730 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/b559f960e314475793d0ee550489fe49 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/b559f960e314475793d0ee550489fe49 2024-11-08T18:40:09,736 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into 70ecfcc2ff1341138ed6f518d3e35977(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:09,736 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:09,736 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=12, startTime=1731091209185; duration=0sec 2024-11-08T18:40:09,737 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:09,737 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:40:09,738 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:09,738 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into b559f960e314475793d0ee550489fe49(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:09,738 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:09,738 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=12, startTime=1731091209186; duration=0sec 2024-11-08T18:40:09,739 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:09,739 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:40:09,740 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50158 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:09,740 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:40:09,740 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:09,740 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/94f0162c256048ff87ce017aafd45e56, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/15c7d8ea0c6c4905bf7a178797faa5fc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/93942675bb81481b832d25b6889ca562, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/831ec16f7986423085b1f58018e1b412] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=49.0 K 2024-11-08T18:40:09,741 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94f0162c256048ff87ce017aafd45e56, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731091205736 2024-11-08T18:40:09,742 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15c7d8ea0c6c4905bf7a178797faa5fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=405, earliestPutTs=1731091206470 2024-11-08T18:40:09,742 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93942675bb81481b832d25b6889ca562, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731091207154 2024-11-08T18:40:09,743 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 831ec16f7986423085b1f58018e1b412, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731091207871 2024-11-08T18:40:09,753 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:09,753 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/4a1393839f554ea092114670cd0008e2 is 50, key is test_row_0/C:col10/1731091207871/Put/seqid=0 2024-11-08T18:40:09,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/efec71725e4c480b85825711dfcdbf5d 2024-11-08T18:40:09,765 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6 is 50, key is test_row_0/C:col10/1731091209063/Put/seqid=0 2024-11-08T18:40:09,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741937_1113 (size=13391) 2024-11-08T18:40:09,777 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/4a1393839f554ea092114670cd0008e2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4a1393839f554ea092114670cd0008e2 2024-11-08T18:40:09,784 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into 4a1393839f554ea092114670cd0008e2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:09,784 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:09,784 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=12, startTime=1731091209186; duration=0sec 2024-11-08T18:40:09,784 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:09,784 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:40:09,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741938_1114 (size=12301) 2024-11-08T18:40:09,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36174 deadline: 1731091269929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:09,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:09,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36144 deadline: 1731091269940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:09,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:10,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6 2024-11-08T18:40:10,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/62dab5d912824fd497d9fd32cf659e05 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62dab5d912824fd497d9fd32cf659e05 2024-11-08T18:40:10,202 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62dab5d912824fd497d9fd32cf659e05, entries=200, sequenceid=458, filesize=14.4 K 2024-11-08T18:40:10,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/efec71725e4c480b85825711dfcdbf5d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/efec71725e4c480b85825711dfcdbf5d 2024-11-08T18:40:10,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/efec71725e4c480b85825711dfcdbf5d, entries=150, sequenceid=458, filesize=12.0 K 2024-11-08T18:40:10,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6 2024-11-08T18:40:10,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6, entries=150, sequenceid=458, filesize=12.0 K 2024-11-08T18:40:10,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1031ms, sequenceid=458, compaction requested=false 2024-11-08T18:40:10,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:10,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:10,442 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:10,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:40:10,442 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-08T18:40:10,442 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:10,443 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:10,443 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:10,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:10,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/163fb8aed0104b31a8e4a7664e68ce58 is 50, key is test_row_0/A:col10/1731091209290/Put/seqid=0 2024-11-08T18:40:10,451 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-08T18:40:10,451 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:10,451 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:10,451 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:10,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:10,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741939_1115 (size=12301) 2024-11-08T18:40:10,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-08T18:40:10,520 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:10,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/163fb8aed0104b31a8e4a7664e68ce58 2024-11-08T18:40:10,873 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/120ef24221e84b5489409e51170c7e71 is 50, key is test_row_0/B:col10/1731091209290/Put/seqid=0 2024-11-08T18:40:10,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741940_1116 (size=12301) 2024-11-08T18:40:11,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/120ef24221e84b5489409e51170c7e71 2024-11-08T18:40:11,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/1ac295931b11470a8629009e5053ff1e is 50, key is test_row_0/C:col10/1731091209290/Put/seqid=0 2024-11-08T18:40:11,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741941_1117 (size=12301) 2024-11-08T18:40:11,637 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T18:40:11,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=485 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/1ac295931b11470a8629009e5053ff1e 2024-11-08T18:40:11,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/163fb8aed0104b31a8e4a7664e68ce58 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/163fb8aed0104b31a8e4a7664e68ce58 2024-11-08T18:40:11,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/163fb8aed0104b31a8e4a7664e68ce58, entries=150, sequenceid=485, filesize=12.0 K 2024-11-08T18:40:11,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/120ef24221e84b5489409e51170c7e71 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/120ef24221e84b5489409e51170c7e71 2024-11-08T18:40:11,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/120ef24221e84b5489409e51170c7e71, entries=150, sequenceid=485, filesize=12.0 K 2024-11-08T18:40:11,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/1ac295931b11470a8629009e5053ff1e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/1ac295931b11470a8629009e5053ff1e 2024-11-08T18:40:11,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/1ac295931b11470a8629009e5053ff1e, entries=150, sequenceid=485, filesize=12.0 K 2024-11-08T18:40:11,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=6.71 KB/6870 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1289ms, sequenceid=485, compaction requested=true 2024-11-08T18:40:11,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:11,732 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:11,732 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store b4a84f4a6ffc3bb08667e2c0f91ea282:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:11,732 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:11,733 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:11,733 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:11,733 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/A is initiating minor compaction (all files) 2024-11-08T18:40:11,733 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/B is initiating minor compaction (all files) 2024-11-08T18:40:11,733 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/A in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:11,733 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/B in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:11,733 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/70ecfcc2ff1341138ed6f518d3e35977, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62dab5d912824fd497d9fd32cf659e05, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/163fb8aed0104b31a8e4a7664e68ce58] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=39.5 K 2024-11-08T18:40:11,733 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/b559f960e314475793d0ee550489fe49, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/efec71725e4c480b85825711dfcdbf5d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/120ef24221e84b5489409e51170c7e71] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=37.1 K 2024-11-08T18:40:11,734 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70ecfcc2ff1341138ed6f518d3e35977, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731091207871 2024-11-08T18:40:11,734 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting b559f960e314475793d0ee550489fe49, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731091207871 2024-11-08T18:40:11,734 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62dab5d912824fd497d9fd32cf659e05, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731091209046 2024-11-08T18:40:11,734 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting efec71725e4c480b85825711dfcdbf5d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731091209061 2024-11-08T18:40:11,734 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 163fb8aed0104b31a8e4a7664e68ce58, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731091209290 2024-11-08T18:40:11,734 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 120ef24221e84b5489409e51170c7e71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731091209290 2024-11-08T18:40:11,744 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#B#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:11,744 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#A#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:11,744 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/3f9fe78825d842b6a16fe163f14f3631 is 50, key is test_row_0/B:col10/1731091209290/Put/seqid=0 2024-11-08T18:40:11,744 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/9379f34d71e3452aa5ab8e0c26c78db9 is 50, key is test_row_0/A:col10/1731091209290/Put/seqid=0 2024-11-08T18:40:11,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741943_1119 (size=13493) 2024-11-08T18:40:11,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741942_1118 (size=13493) 2024-11-08T18:40:12,165 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/9379f34d71e3452aa5ab8e0c26c78db9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9379f34d71e3452aa5ab8e0c26c78db9 2024-11-08T18:40:12,165 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/3f9fe78825d842b6a16fe163f14f3631 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3f9fe78825d842b6a16fe163f14f3631 2024-11-08T18:40:12,171 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/A of b4a84f4a6ffc3bb08667e2c0f91ea282 into 9379f34d71e3452aa5ab8e0c26c78db9(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:12,171 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/B of b4a84f4a6ffc3bb08667e2c0f91ea282 into 3f9fe78825d842b6a16fe163f14f3631(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:12,171 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:12,171 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:12,171 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/A, priority=13, startTime=1731091211732; duration=0sec 2024-11-08T18:40:12,171 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/B, priority=13, startTime=1731091211732; duration=0sec 2024-11-08T18:40:12,172 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:12,172 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:A 2024-11-08T18:40:12,172 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:12,172 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:12,172 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:B 2024-11-08T18:40:12,173 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:12,173 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): b4a84f4a6ffc3bb08667e2c0f91ea282/C is initiating minor compaction (all files) 2024-11-08T18:40:12,173 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of b4a84f4a6ffc3bb08667e2c0f91ea282/C in TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:12,173 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4a1393839f554ea092114670cd0008e2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/1ac295931b11470a8629009e5053ff1e] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp, totalSize=37.1 K 2024-11-08T18:40:12,173 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a1393839f554ea092114670cd0008e2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=445, earliestPutTs=1731091207871 2024-11-08T18:40:12,174 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d8f872ca7a946c8aaa0e65bb0aa1cc6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731091209061 2024-11-08T18:40:12,174 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ac295931b11470a8629009e5053ff1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=485, earliestPutTs=1731091209290 2024-11-08T18:40:12,186 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): b4a84f4a6ffc3bb08667e2c0f91ea282#C#compaction#107 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:12,186 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/c8e919fe6ff442ff8bbf03ecd4171921 is 50, key is test_row_0/C:col10/1731091209290/Put/seqid=0 2024-11-08T18:40:12,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741944_1120 (size=13493) 2024-11-08T18:40:12,605 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/c8e919fe6ff442ff8bbf03ecd4171921 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c8e919fe6ff442ff8bbf03ecd4171921 2024-11-08T18:40:12,612 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in b4a84f4a6ffc3bb08667e2c0f91ea282/C of b4a84f4a6ffc3bb08667e2c0f91ea282 into c8e919fe6ff442ff8bbf03ecd4171921(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:12,612 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for b4a84f4a6ffc3bb08667e2c0f91ea282: 2024-11-08T18:40:12,612 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., storeName=b4a84f4a6ffc3bb08667e2c0f91ea282/C, priority=13, startTime=1731091211732; duration=0sec 2024-11-08T18:40:12,612 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:12,612 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: b4a84f4a6ffc3bb08667e2c0f91ea282:C 2024-11-08T18:40:16,251 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-08T18:40:16,251 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:16,251 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:16,251 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:16,252 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:16,282 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-08T18:40:16,282 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:16,282 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:16,282 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:16,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:16,321 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-08T18:40:16,321 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:16,321 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:16,321 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:16,321 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 57 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 106 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2967 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2904 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T18:40:16,321 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1386 2024-11-08T18:40:16,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4157 rows 2024-11-08T18:40:16,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1388 2024-11-08T18:40:16,322 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4154 rows 2024-11-08T18:40:16,322 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T18:40:16,322 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:40:16,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:16,322 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:16,322 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:16,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T18:40:16,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:16,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T18:40:16,333 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091216333"}]},"ts":"1731091216333"} 2024-11-08T18:40:16,335 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T18:40:16,335 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T18:40:16,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:40:16,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, UNASSIGN}] 2024-11-08T18:40:16,345 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, UNASSIGN 2024-11-08T18:40:16,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b4a84f4a6ffc3bb08667e2c0f91ea282, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:16,349 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, UNASSIGN because future has completed 2024-11-08T18:40:16,349 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:40:16,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:40:16,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T18:40:16,508 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:16,509 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:40:16,510 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing b4a84f4a6ffc3bb08667e2c0f91ea282, disabling compactions & flushes 2024-11-08T18:40:16,510 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:16,510 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:16,510 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. after waiting 0 ms 2024-11-08T18:40:16,510 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:16,510 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing b4a84f4a6ffc3bb08667e2c0f91ea282 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-08T18:40:16,522 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/dc5d36e8d13f480f999d59030b64a012 is 50, key is test_row_2/A:col10/1731091216319/Put/seqid=0 2024-11-08T18:40:16,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741945_1121 (size=7415) 2024-11-08T18:40:16,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T18:40:16,927 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/dc5d36e8d13f480f999d59030b64a012 2024-11-08T18:40:16,940 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/282c8b44fed34ec49034145e7bd8c2fc is 50, key is test_row_2/B:col10/1731091216319/Put/seqid=0 2024-11-08T18:40:16,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741946_1122 (size=7415) 2024-11-08T18:40:16,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T18:40:17,345 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/282c8b44fed34ec49034145e7bd8c2fc 2024-11-08T18:40:17,361 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/82ab16350fe54121bec290aa7e13cc6e is 50, key is test_row_2/C:col10/1731091216319/Put/seqid=0 2024-11-08T18:40:17,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741947_1123 (size=7415) 2024-11-08T18:40:17,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T18:40:17,767 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=495 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/82ab16350fe54121bec290aa7e13cc6e 2024-11-08T18:40:17,772 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/A/dc5d36e8d13f480f999d59030b64a012 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/dc5d36e8d13f480f999d59030b64a012 2024-11-08T18:40:17,777 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/dc5d36e8d13f480f999d59030b64a012, entries=50, sequenceid=495, filesize=7.2 K 2024-11-08T18:40:17,778 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/B/282c8b44fed34ec49034145e7bd8c2fc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/282c8b44fed34ec49034145e7bd8c2fc 2024-11-08T18:40:17,783 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/282c8b44fed34ec49034145e7bd8c2fc, entries=50, sequenceid=495, filesize=7.2 K 2024-11-08T18:40:17,784 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/.tmp/C/82ab16350fe54121bec290aa7e13cc6e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/82ab16350fe54121bec290aa7e13cc6e 2024-11-08T18:40:17,788 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/82ab16350fe54121bec290aa7e13cc6e, entries=50, sequenceid=495, filesize=7.2 K 2024-11-08T18:40:17,789 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1279ms, sequenceid=495, compaction requested=false 2024-11-08T18:40:17,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5a7ef95d346b49369c322b9f1667d165, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9ef789d322c74fdf8f8832b28caa1342, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2daf14c683824f628165f0d361d4e829, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e4ac323819e840668e21abae5586d2a9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/6cc9f304068c4f72adee7fdad6c31674, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/8be5ede66b2940d78fecb9725cbeebcc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/31f81d43ca8c45429fe266ff69c816fc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/54e6c1f0f8864a08b805a2f4081aed78, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62f69fecf08e48cd9694127a6e8e31ba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4863f7a93fd74b44b5123ddc6c91d6c5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5973bfbb5b294721b4a47e447ff610ed, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e9683b8c1bd54abbae545af08350ec5c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/05014fb754d54c84abfc914dede45e59, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a894f16dd01a43f39f57301732b58774, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/adb0fad71bbb4a0cad967c247fb1078a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7cef6bb64a724f19b7afa79b13bdeb06, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a5cdf3a028194dbc8f974016af665ace, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3de322453215432b84e4d5680ea8d87d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2ed4c76904904c4291ba476c1b575d10, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4a9ab506ec814db38432194ff9c5ae92, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7937aff794a84f4d8fe083870bf220ae, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/369327c6200d4f4c8662970b7d7241cc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3d9c6dee3062424e9905c97e9c3b8fa4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/661d2b153f10402984baf531b929649a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/08087200554c4476956bc7dc95cfeaad, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/b63a73df0b6e46feb65b052e8e954ef7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e5894beb62224b5ea4b138effe03bc04, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/cbbf042ec3c0428dbdb5afd506b1c9db, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4451ab5db86d4d548d859a7b93cdf921, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2f4ca6020df44586afbc9b76f3fd5729, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/86c5b0ddf4cc4cf9840dd085cf1b1044, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/ac800afb61f24ae380bbb997a462a15f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/70ecfcc2ff1341138ed6f518d3e35977, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62dab5d912824fd497d9fd32cf659e05, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/163fb8aed0104b31a8e4a7664e68ce58] to archive 2024-11-08T18:40:17,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:40:17,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5a7ef95d346b49369c322b9f1667d165 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5a7ef95d346b49369c322b9f1667d165 2024-11-08T18:40:17,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9ef789d322c74fdf8f8832b28caa1342 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9ef789d322c74fdf8f8832b28caa1342 2024-11-08T18:40:17,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2daf14c683824f628165f0d361d4e829 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2daf14c683824f628165f0d361d4e829 2024-11-08T18:40:17,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e4ac323819e840668e21abae5586d2a9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e4ac323819e840668e21abae5586d2a9 2024-11-08T18:40:17,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/6cc9f304068c4f72adee7fdad6c31674 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/6cc9f304068c4f72adee7fdad6c31674 2024-11-08T18:40:17,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/8be5ede66b2940d78fecb9725cbeebcc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/8be5ede66b2940d78fecb9725cbeebcc 2024-11-08T18:40:17,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/31f81d43ca8c45429fe266ff69c816fc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/31f81d43ca8c45429fe266ff69c816fc 2024-11-08T18:40:17,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/54e6c1f0f8864a08b805a2f4081aed78 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/54e6c1f0f8864a08b805a2f4081aed78 2024-11-08T18:40:17,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62f69fecf08e48cd9694127a6e8e31ba to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62f69fecf08e48cd9694127a6e8e31ba 2024-11-08T18:40:17,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4863f7a93fd74b44b5123ddc6c91d6c5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4863f7a93fd74b44b5123ddc6c91d6c5 2024-11-08T18:40:17,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5973bfbb5b294721b4a47e447ff610ed to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/5973bfbb5b294721b4a47e447ff610ed 2024-11-08T18:40:17,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e9683b8c1bd54abbae545af08350ec5c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e9683b8c1bd54abbae545af08350ec5c 2024-11-08T18:40:17,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/05014fb754d54c84abfc914dede45e59 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/05014fb754d54c84abfc914dede45e59 2024-11-08T18:40:17,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a894f16dd01a43f39f57301732b58774 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a894f16dd01a43f39f57301732b58774 2024-11-08T18:40:17,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/adb0fad71bbb4a0cad967c247fb1078a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/adb0fad71bbb4a0cad967c247fb1078a 2024-11-08T18:40:17,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7cef6bb64a724f19b7afa79b13bdeb06 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7cef6bb64a724f19b7afa79b13bdeb06 2024-11-08T18:40:17,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a5cdf3a028194dbc8f974016af665ace to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/a5cdf3a028194dbc8f974016af665ace 2024-11-08T18:40:17,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3de322453215432b84e4d5680ea8d87d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3de322453215432b84e4d5680ea8d87d 2024-11-08T18:40:17,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2ed4c76904904c4291ba476c1b575d10 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2ed4c76904904c4291ba476c1b575d10 2024-11-08T18:40:17,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4a9ab506ec814db38432194ff9c5ae92 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4a9ab506ec814db38432194ff9c5ae92 2024-11-08T18:40:17,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7937aff794a84f4d8fe083870bf220ae to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/7937aff794a84f4d8fe083870bf220ae 2024-11-08T18:40:17,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/369327c6200d4f4c8662970b7d7241cc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/369327c6200d4f4c8662970b7d7241cc 2024-11-08T18:40:17,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3d9c6dee3062424e9905c97e9c3b8fa4 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/3d9c6dee3062424e9905c97e9c3b8fa4 2024-11-08T18:40:17,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/661d2b153f10402984baf531b929649a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/661d2b153f10402984baf531b929649a 2024-11-08T18:40:17,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/08087200554c4476956bc7dc95cfeaad to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/08087200554c4476956bc7dc95cfeaad 2024-11-08T18:40:17,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/b63a73df0b6e46feb65b052e8e954ef7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/b63a73df0b6e46feb65b052e8e954ef7 2024-11-08T18:40:17,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e5894beb62224b5ea4b138effe03bc04 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/e5894beb62224b5ea4b138effe03bc04 2024-11-08T18:40:17,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/cbbf042ec3c0428dbdb5afd506b1c9db to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/cbbf042ec3c0428dbdb5afd506b1c9db 2024-11-08T18:40:17,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4451ab5db86d4d548d859a7b93cdf921 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/4451ab5db86d4d548d859a7b93cdf921 2024-11-08T18:40:17,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2f4ca6020df44586afbc9b76f3fd5729 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/2f4ca6020df44586afbc9b76f3fd5729 2024-11-08T18:40:17,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/86c5b0ddf4cc4cf9840dd085cf1b1044 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/86c5b0ddf4cc4cf9840dd085cf1b1044 2024-11-08T18:40:17,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/ac800afb61f24ae380bbb997a462a15f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/ac800afb61f24ae380bbb997a462a15f 2024-11-08T18:40:17,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/70ecfcc2ff1341138ed6f518d3e35977 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/70ecfcc2ff1341138ed6f518d3e35977 2024-11-08T18:40:17,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62dab5d912824fd497d9fd32cf659e05 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/62dab5d912824fd497d9fd32cf659e05 2024-11-08T18:40:17,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/163fb8aed0104b31a8e4a7664e68ce58 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/163fb8aed0104b31a8e4a7664e68ce58 2024-11-08T18:40:17,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/54b5e16e71824902a4c2ed75b8caadb9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9aef3500845a4bf0ab27ea6c6eefbf43, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/cf9704ca2c76470589d8b4d1d481c576, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a19d172f478f4544a3027962ee0d493b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/2d930470ca394eb4b437abccb5e5812e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/80bdd3bae6144dedb16d92a5b0f3aa14, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/7b3ffbca2fbf4478aed0c0082882229a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9befce67a21e4cc78e395b175b5c1dc9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/e4081383fd2b402dae898f6af8f3f40f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/372da6f1a72647f9bdaa8f284896c6ad, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/677f18f931704841bcf03aa2e618ce35, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f4f12d1a02434123889f96582a79e800, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/527bcdebe6b24404a74fd27d2652ca52, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/705018cc4972447e95a4f2dfe6a5143d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1efb1634bfbb49608b4fed409e7bb1e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/eecf574f60a642e9a0c0d7496235e670, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a86c4a40b8564f15b586e63f66164d95, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9b089f69358942128aad825c095eab62, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1b92075652ff4bb5802251fd5514f89c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/49587dd57479460baf316a83abcd9429, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ba4cca8a6de4f50998774cb975027a7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/161d35463453454da35424288204145b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a0ae632bc922422292ecbcce8d839e1c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/203eb5c3c7984970b9c3b9d5032a332e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/402e5f40242547e5b1f22471e39521cf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ad8038328c94d23956fda6ced03d0e8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/8c61b43c60df4441a954900333208d6e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f6b5cd37e6ac42e595227406f93bd6b7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/24e591dddb6747498ca809711fe71c06, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/75bb820df4da4376b077cd107577cb6e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9d3ccca73b1b49458e2f007d50f9f314, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/b559f960e314475793d0ee550489fe49, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1081cfbf8f934fa7b053718455ee8e80, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/efec71725e4c480b85825711dfcdbf5d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/120ef24221e84b5489409e51170c7e71] to archive 2024-11-08T18:40:17,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:40:17,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/54b5e16e71824902a4c2ed75b8caadb9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/54b5e16e71824902a4c2ed75b8caadb9 2024-11-08T18:40:17,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9aef3500845a4bf0ab27ea6c6eefbf43 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9aef3500845a4bf0ab27ea6c6eefbf43 2024-11-08T18:40:17,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/cf9704ca2c76470589d8b4d1d481c576 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/cf9704ca2c76470589d8b4d1d481c576 2024-11-08T18:40:17,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a19d172f478f4544a3027962ee0d493b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a19d172f478f4544a3027962ee0d493b 2024-11-08T18:40:17,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/2d930470ca394eb4b437abccb5e5812e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/2d930470ca394eb4b437abccb5e5812e 2024-11-08T18:40:17,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/80bdd3bae6144dedb16d92a5b0f3aa14 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/80bdd3bae6144dedb16d92a5b0f3aa14 2024-11-08T18:40:17,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/7b3ffbca2fbf4478aed0c0082882229a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/7b3ffbca2fbf4478aed0c0082882229a 2024-11-08T18:40:17,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9befce67a21e4cc78e395b175b5c1dc9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9befce67a21e4cc78e395b175b5c1dc9 2024-11-08T18:40:17,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/e4081383fd2b402dae898f6af8f3f40f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/e4081383fd2b402dae898f6af8f3f40f 2024-11-08T18:40:17,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/372da6f1a72647f9bdaa8f284896c6ad to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/372da6f1a72647f9bdaa8f284896c6ad 2024-11-08T18:40:17,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/677f18f931704841bcf03aa2e618ce35 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/677f18f931704841bcf03aa2e618ce35 2024-11-08T18:40:17,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f4f12d1a02434123889f96582a79e800 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f4f12d1a02434123889f96582a79e800 2024-11-08T18:40:17,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/527bcdebe6b24404a74fd27d2652ca52 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/527bcdebe6b24404a74fd27d2652ca52 2024-11-08T18:40:17,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/705018cc4972447e95a4f2dfe6a5143d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/705018cc4972447e95a4f2dfe6a5143d 2024-11-08T18:40:17,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1efb1634bfbb49608b4fed409e7bb1e3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1efb1634bfbb49608b4fed409e7bb1e3 2024-11-08T18:40:17,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/eecf574f60a642e9a0c0d7496235e670 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/eecf574f60a642e9a0c0d7496235e670 2024-11-08T18:40:17,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a86c4a40b8564f15b586e63f66164d95 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a86c4a40b8564f15b586e63f66164d95 2024-11-08T18:40:17,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9b089f69358942128aad825c095eab62 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9b089f69358942128aad825c095eab62 2024-11-08T18:40:17,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1b92075652ff4bb5802251fd5514f89c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1b92075652ff4bb5802251fd5514f89c 2024-11-08T18:40:17,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/49587dd57479460baf316a83abcd9429 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/49587dd57479460baf316a83abcd9429 2024-11-08T18:40:17,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ba4cca8a6de4f50998774cb975027a7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ba4cca8a6de4f50998774cb975027a7 2024-11-08T18:40:17,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/161d35463453454da35424288204145b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/161d35463453454da35424288204145b 2024-11-08T18:40:17,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a0ae632bc922422292ecbcce8d839e1c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/a0ae632bc922422292ecbcce8d839e1c 2024-11-08T18:40:17,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/203eb5c3c7984970b9c3b9d5032a332e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/203eb5c3c7984970b9c3b9d5032a332e 2024-11-08T18:40:17,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/402e5f40242547e5b1f22471e39521cf to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/402e5f40242547e5b1f22471e39521cf 2024-11-08T18:40:17,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ad8038328c94d23956fda6ced03d0e8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3ad8038328c94d23956fda6ced03d0e8 2024-11-08T18:40:17,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/8c61b43c60df4441a954900333208d6e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/8c61b43c60df4441a954900333208d6e 2024-11-08T18:40:17,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f6b5cd37e6ac42e595227406f93bd6b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/f6b5cd37e6ac42e595227406f93bd6b7 2024-11-08T18:40:17,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/24e591dddb6747498ca809711fe71c06 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/24e591dddb6747498ca809711fe71c06 2024-11-08T18:40:17,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/75bb820df4da4376b077cd107577cb6e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/75bb820df4da4376b077cd107577cb6e 2024-11-08T18:40:17,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9d3ccca73b1b49458e2f007d50f9f314 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/9d3ccca73b1b49458e2f007d50f9f314 2024-11-08T18:40:17,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/b559f960e314475793d0ee550489fe49 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/b559f960e314475793d0ee550489fe49 2024-11-08T18:40:17,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1081cfbf8f934fa7b053718455ee8e80 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/1081cfbf8f934fa7b053718455ee8e80 2024-11-08T18:40:17,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/efec71725e4c480b85825711dfcdbf5d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/efec71725e4c480b85825711dfcdbf5d 2024-11-08T18:40:17,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/120ef24221e84b5489409e51170c7e71 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/120ef24221e84b5489409e51170c7e71 2024-11-08T18:40:17,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/803ef8c2544e493f8267f28a01e8fe04, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/77b313e7e2824472bcbf8af8b630a22e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/78a5ef9cc1924238a80fd07870c62776, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c3097cd4d6284b3cb1e0ea26aa2fce80, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/70ae62abc699413889c4ea877c3508a6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/791ceff19f7e4413b9deca73c0b4e893, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/37d6f44be51d4401b0ce35f161b1927d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef55d74db11d4c0fb432f53cfb7f8705, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/2c95c73c5776405d894457f7ca3f384d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/b4196be86593414baaa28024c51685d7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/92fe07b842f849cfbe5b65a7a7297c44, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/0d64d2dbb0124c7db91ceafb8d481f7f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/d96249457cce46c9aa61868427643f9c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/46c9cb6c11f44d1b8a38732fbbf571bd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/131f7b54383c40d6bd529cad19f0882d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/fed3b04206dd466d9207105e8f8923bb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3e23e3b716e54d45b56d18c7ecea5508, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/18020bc61b194f678d641768ed9663f8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8918dfac7d154cb68bf5a9a9f52ec50a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8e275b5d52764e159f7cb41622f008e6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c321d58f930e4e30a11e27b2bda53ff6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/17dd23c37d8f46f2841c9fca70dab3d0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ae1bb99f016c403e9c4b98eac3a0258f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef2fba2358c640cf8565b795f5d8022c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bb55c04e4a4a43109609d90939f4e9e0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bfd7c0e091b34123bba867f56893007b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4244ec6ddacd4b3185e708d0b07956a8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/94f0162c256048ff87ce017aafd45e56, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/85801b7698364e06abe1b61c78a283fa, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/15c7d8ea0c6c4905bf7a178797faa5fc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/93942675bb81481b832d25b6889ca562, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4a1393839f554ea092114670cd0008e2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/831ec16f7986423085b1f58018e1b412, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/1ac295931b11470a8629009e5053ff1e] to archive 2024-11-08T18:40:17,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:40:17,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/803ef8c2544e493f8267f28a01e8fe04 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/803ef8c2544e493f8267f28a01e8fe04 2024-11-08T18:40:17,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/77b313e7e2824472bcbf8af8b630a22e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/77b313e7e2824472bcbf8af8b630a22e 2024-11-08T18:40:17,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/78a5ef9cc1924238a80fd07870c62776 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/78a5ef9cc1924238a80fd07870c62776 2024-11-08T18:40:17,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c3097cd4d6284b3cb1e0ea26aa2fce80 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c3097cd4d6284b3cb1e0ea26aa2fce80 2024-11-08T18:40:17,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/70ae62abc699413889c4ea877c3508a6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/70ae62abc699413889c4ea877c3508a6 2024-11-08T18:40:17,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/791ceff19f7e4413b9deca73c0b4e893 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/791ceff19f7e4413b9deca73c0b4e893 2024-11-08T18:40:17,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/37d6f44be51d4401b0ce35f161b1927d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/37d6f44be51d4401b0ce35f161b1927d 2024-11-08T18:40:17,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef55d74db11d4c0fb432f53cfb7f8705 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef55d74db11d4c0fb432f53cfb7f8705 2024-11-08T18:40:17,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/2c95c73c5776405d894457f7ca3f384d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/2c95c73c5776405d894457f7ca3f384d 2024-11-08T18:40:17,933 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/b4196be86593414baaa28024c51685d7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/b4196be86593414baaa28024c51685d7 2024-11-08T18:40:17,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/92fe07b842f849cfbe5b65a7a7297c44 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/92fe07b842f849cfbe5b65a7a7297c44 2024-11-08T18:40:17,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/0d64d2dbb0124c7db91ceafb8d481f7f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/0d64d2dbb0124c7db91ceafb8d481f7f 2024-11-08T18:40:17,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/d96249457cce46c9aa61868427643f9c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/d96249457cce46c9aa61868427643f9c 2024-11-08T18:40:17,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/46c9cb6c11f44d1b8a38732fbbf571bd to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/46c9cb6c11f44d1b8a38732fbbf571bd 2024-11-08T18:40:17,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/131f7b54383c40d6bd529cad19f0882d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/131f7b54383c40d6bd529cad19f0882d 2024-11-08T18:40:17,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/fed3b04206dd466d9207105e8f8923bb to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/fed3b04206dd466d9207105e8f8923bb 2024-11-08T18:40:17,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3e23e3b716e54d45b56d18c7ecea5508 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3e23e3b716e54d45b56d18c7ecea5508 2024-11-08T18:40:17,943 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/18020bc61b194f678d641768ed9663f8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/18020bc61b194f678d641768ed9663f8 2024-11-08T18:40:17,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8918dfac7d154cb68bf5a9a9f52ec50a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8918dfac7d154cb68bf5a9a9f52ec50a 2024-11-08T18:40:17,945 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8e275b5d52764e159f7cb41622f008e6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/8e275b5d52764e159f7cb41622f008e6 2024-11-08T18:40:17,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c321d58f930e4e30a11e27b2bda53ff6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c321d58f930e4e30a11e27b2bda53ff6 2024-11-08T18:40:17,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/17dd23c37d8f46f2841c9fca70dab3d0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/17dd23c37d8f46f2841c9fca70dab3d0 2024-11-08T18:40:17,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ae1bb99f016c403e9c4b98eac3a0258f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ae1bb99f016c403e9c4b98eac3a0258f 2024-11-08T18:40:17,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef2fba2358c640cf8565b795f5d8022c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/ef2fba2358c640cf8565b795f5d8022c 2024-11-08T18:40:17,952 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bb55c04e4a4a43109609d90939f4e9e0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bb55c04e4a4a43109609d90939f4e9e0 2024-11-08T18:40:17,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bfd7c0e091b34123bba867f56893007b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/bfd7c0e091b34123bba867f56893007b 2024-11-08T18:40:17,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4244ec6ddacd4b3185e708d0b07956a8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4244ec6ddacd4b3185e708d0b07956a8 2024-11-08T18:40:17,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/94f0162c256048ff87ce017aafd45e56 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/94f0162c256048ff87ce017aafd45e56 2024-11-08T18:40:17,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/85801b7698364e06abe1b61c78a283fa to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/85801b7698364e06abe1b61c78a283fa 2024-11-08T18:40:17,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/15c7d8ea0c6c4905bf7a178797faa5fc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/15c7d8ea0c6c4905bf7a178797faa5fc 2024-11-08T18:40:17,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/93942675bb81481b832d25b6889ca562 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/93942675bb81481b832d25b6889ca562 2024-11-08T18:40:17,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4a1393839f554ea092114670cd0008e2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/4a1393839f554ea092114670cd0008e2 2024-11-08T18:40:17,962 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/831ec16f7986423085b1f58018e1b412 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/831ec16f7986423085b1f58018e1b412 2024-11-08T18:40:17,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/3d8f872ca7a946c8aaa0e65bb0aa1cc6 2024-11-08T18:40:17,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/1ac295931b11470a8629009e5053ff1e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/1ac295931b11470a8629009e5053ff1e 2024-11-08T18:40:17,970 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/recovered.edits/498.seqid, newMaxSeqId=498, maxSeqId=1 2024-11-08T18:40:17,971 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. 2024-11-08T18:40:17,971 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for b4a84f4a6ffc3bb08667e2c0f91ea282: Waiting for close lock at 1731091216510Running coprocessor pre-close hooks at 1731091216510Disabling compacts and flushes for region at 1731091216510Disabling writes for close at 1731091216510Obtaining lock to block concurrent updates at 1731091216510Preparing flush snapshotting stores in b4a84f4a6ffc3bb08667e2c0f91ea282 at 1731091216510Finished memstore snapshotting TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731091216511 (+1 ms)Flushing stores of TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282. at 1731091216512 (+1 ms)Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/A: creating writer at 1731091216512Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/A: appending metadata at 1731091216520 (+8 ms)Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/A: closing flushed file at 1731091216521 (+1 ms)Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/B: creating writer at 1731091216933 (+412 ms)Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/B: appending metadata at 1731091216939 (+6 ms)Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/B: closing flushed file at 1731091216939Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/C: creating writer at 1731091217350 (+411 ms)Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/C: appending metadata at 1731091217360 (+10 ms)Flushing b4a84f4a6ffc3bb08667e2c0f91ea282/C: closing flushed file at 1731091217360Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e88aac4: reopening flushed file at 1731091217771 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@31b98c5c: reopening flushed file at 1731091217777 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59a32a95: reopening flushed file at 1731091217783 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for b4a84f4a6ffc3bb08667e2c0f91ea282 in 1279ms, sequenceid=495, compaction requested=false at 1731091217789 (+6 ms)Writing region close event to WAL at 1731091217966 (+177 ms)Running coprocessor post-close hooks at 1731091217971 (+5 ms)Closed at 1731091217971 2024-11-08T18:40:17,974 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:17,974 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=b4a84f4a6ffc3bb08667e2c0f91ea282, regionState=CLOSED 2024-11-08T18:40:17,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:40:17,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-08T18:40:17,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure b4a84f4a6ffc3bb08667e2c0f91ea282, server=b75fe890c219,44447,1731091183487 in 1.6270 sec 2024-11-08T18:40:17,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-08T18:40:17,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=b4a84f4a6ffc3bb08667e2c0f91ea282, UNASSIGN in 1.6360 sec 2024-11-08T18:40:17,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-08T18:40:17,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6450 sec 2024-11-08T18:40:17,985 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091217985"}]},"ts":"1731091217985"} 2024-11-08T18:40:17,987 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T18:40:17,987 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T18:40:17,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6610 sec 2024-11-08T18:40:18,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-08T18:40:18,480 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:18,483 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T18:40:18,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:18,487 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:18,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T18:40:18,489 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:18,494 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:18,498 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/recovered.edits] 2024-11-08T18:40:18,502 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9379f34d71e3452aa5ab8e0c26c78db9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/9379f34d71e3452aa5ab8e0c26c78db9 2024-11-08T18:40:18,503 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/dc5d36e8d13f480f999d59030b64a012 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/A/dc5d36e8d13f480f999d59030b64a012 2024-11-08T18:40:18,506 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/282c8b44fed34ec49034145e7bd8c2fc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/282c8b44fed34ec49034145e7bd8c2fc 2024-11-08T18:40:18,507 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3f9fe78825d842b6a16fe163f14f3631 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/B/3f9fe78825d842b6a16fe163f14f3631 2024-11-08T18:40:18,510 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/82ab16350fe54121bec290aa7e13cc6e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/82ab16350fe54121bec290aa7e13cc6e 2024-11-08T18:40:18,512 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c8e919fe6ff442ff8bbf03ecd4171921 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/C/c8e919fe6ff442ff8bbf03ecd4171921 2024-11-08T18:40:18,515 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/recovered.edits/498.seqid to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282/recovered.edits/498.seqid 2024-11-08T18:40:18,516 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/b4a84f4a6ffc3bb08667e2c0f91ea282 2024-11-08T18:40:18,516 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T18:40:18,522 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:18,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-08T18:40:18,533 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T18:40:18,570 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T18:40:18,572 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:18,573 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T18:40:18,573 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731091218573"}]},"ts":"9223372036854775807"} 2024-11-08T18:40:18,579 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T18:40:18,579 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b4a84f4a6ffc3bb08667e2c0f91ea282, NAME => 'TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T18:40:18,579 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T18:40:18,580 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731091218580"}]},"ts":"9223372036854775807"} 2024-11-08T18:40:18,584 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T18:40:18,585 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:18,587 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 102 msec 2024-11-08T18:40:18,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-08T18:40:18,601 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T18:40:18,601 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:18,615 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;b75fe890c219:44447-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/b75fe890c219:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_733759443_22 at /127.0.0.1:59386 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=456 (was 303) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4777 (was 5010) 2024-11-08T18:40:18,632 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=456, ProcessCount=11, AvailableMemoryMB=4774 2024-11-08T18:40:18,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:40:18,634 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:40:18,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:18,637 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T18:40:18,637 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:18,638 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-08T18:40:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T18:40:18,639 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T18:40:18,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741948_1124 (size=959) 2024-11-08T18:40:18,653 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 2024-11-08T18:40:18,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741949_1125 (size=53) 2024-11-08T18:40:18,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T18:40:18,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T18:40:19,071 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:40:19,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2db46d8a8e52bdcd84049ec79d0d81b7, disabling compactions & flushes 2024-11-08T18:40:19,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:19,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:19,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. after waiting 0 ms 2024-11-08T18:40:19,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:19,072 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:19,072 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2db46d8a8e52bdcd84049ec79d0d81b7: Waiting for close lock at 1731091219071Disabling compacts and flushes for region at 1731091219071Disabling writes for close at 1731091219072 (+1 ms)Writing region close event to WAL at 1731091219072Closed at 1731091219072 2024-11-08T18:40:19,075 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T18:40:19,076 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731091219075"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731091219075"}]},"ts":"1731091219075"} 2024-11-08T18:40:19,078 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T18:40:19,080 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T18:40:19,080 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091219080"}]},"ts":"1731091219080"} 2024-11-08T18:40:19,082 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T18:40:19,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, ASSIGN}] 2024-11-08T18:40:19,084 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, ASSIGN 2024-11-08T18:40:19,085 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, ASSIGN; state=OFFLINE, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=false 2024-11-08T18:40:19,236 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2db46d8a8e52bdcd84049ec79d0d81b7, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:19,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, ASSIGN because future has completed 2024-11-08T18:40:19,239 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:40:19,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T18:40:19,396 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:19,396 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:40:19,396 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,396 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:40:19,397 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,397 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,399 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,402 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2db46d8a8e52bdcd84049ec79d0d81b7 columnFamilyName A 2024-11-08T18:40:19,402 DEBUG [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:19,403 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(327): Store=2db46d8a8e52bdcd84049ec79d0d81b7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:19,404 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,405 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2db46d8a8e52bdcd84049ec79d0d81b7 columnFamilyName B 2024-11-08T18:40:19,406 DEBUG [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:19,406 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(327): Store=2db46d8a8e52bdcd84049ec79d0d81b7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:19,406 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,408 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2db46d8a8e52bdcd84049ec79d0d81b7 columnFamilyName C 2024-11-08T18:40:19,408 DEBUG [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:19,408 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(327): Store=2db46d8a8e52bdcd84049ec79d0d81b7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:19,408 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,409 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,409 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,410 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,410 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,411 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:40:19,412 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,414 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T18:40:19,415 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 2db46d8a8e52bdcd84049ec79d0d81b7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64117279, jitterRate=-0.04457809031009674}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:40:19,415 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:19,415 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 2db46d8a8e52bdcd84049ec79d0d81b7: Running coprocessor pre-open hook at 1731091219397Writing region info on filesystem at 1731091219397Initializing all the Stores at 1731091219398 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091219398Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091219398Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091219398Cleaning up temporary data from old regions at 1731091219410 (+12 ms)Running coprocessor post-open hooks at 1731091219415 (+5 ms)Region opened successfully at 1731091219415 2024-11-08T18:40:19,416 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., pid=34, masterSystemTime=1731091219391 2024-11-08T18:40:19,418 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:19,418 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:19,419 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=2db46d8a8e52bdcd84049ec79d0d81b7, regionState=OPEN, openSeqNum=2, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:19,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:40:19,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-08T18:40:19,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 in 182 msec 2024-11-08T18:40:19,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-08T18:40:19,425 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, ASSIGN in 341 msec 2024-11-08T18:40:19,425 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T18:40:19,426 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091219426"}]},"ts":"1731091219426"} 2024-11-08T18:40:19,427 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T18:40:19,428 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T18:40:19,430 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 794 msec 2024-11-08T18:40:19,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-08T18:40:19,780 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:19,781 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@338737eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:19,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:19,781 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:19,782 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:19,782 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6054ef04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:19,783 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:19,784 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a1c705, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:19,785 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:19,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:19,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:19,789 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T18:40:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is b75fe890c219,42293,1731091182659 2024-11-08T18:40:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@234e638 2024-11-08T18:40:19,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T18:40:19,791 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T18:40:19,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:40:19,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:40:19,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:19,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741950_1126 (size=995) 2024-11-08T18:40:20,219 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T18:40:20,219 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T18:40:20,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:40:20,228 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, REOPEN/MOVE}] 2024-11-08T18:40:20,229 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, REOPEN/MOVE 2024-11-08T18:40:20,230 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=2db46d8a8e52bdcd84049ec79d0d81b7, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:20,232 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, REOPEN/MOVE because future has completed 2024-11-08T18:40:20,232 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:40:20,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:40:20,385 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,385 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:40:20,385 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 2db46d8a8e52bdcd84049ec79d0d81b7, disabling compactions & flushes 2024-11-08T18:40:20,385 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,385 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,385 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. after waiting 0 ms 2024-11-08T18:40:20,385 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,390 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T18:40:20,390 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,390 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 2db46d8a8e52bdcd84049ec79d0d81b7: Waiting for close lock at 1731091220385Running coprocessor pre-close hooks at 1731091220385Disabling compacts and flushes for region at 1731091220385Disabling writes for close at 1731091220385Writing region close event to WAL at 1731091220386 (+1 ms)Running coprocessor post-close hooks at 1731091220390 (+4 ms)Closed at 1731091220390 2024-11-08T18:40:20,391 WARN [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 2db46d8a8e52bdcd84049ec79d0d81b7 to self. 2024-11-08T18:40:20,393 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,393 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=2db46d8a8e52bdcd84049ec79d0d81b7, regionState=CLOSED 2024-11-08T18:40:20,395 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:40:20,397 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-08T18:40:20,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 in 164 msec 2024-11-08T18:40:20,398 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, REOPEN/MOVE; state=CLOSED, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=true 2024-11-08T18:40:20,549 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=2db46d8a8e52bdcd84049ec79d0d81b7, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:20,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, REOPEN/MOVE because future has completed 2024-11-08T18:40:20,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:40:20,707 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,707 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:40:20,707 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,707 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:40:20,707 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,707 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,711 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,716 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2db46d8a8e52bdcd84049ec79d0d81b7 columnFamilyName A 2024-11-08T18:40:20,718 DEBUG [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:20,718 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(327): Store=2db46d8a8e52bdcd84049ec79d0d81b7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:20,719 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,720 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2db46d8a8e52bdcd84049ec79d0d81b7 columnFamilyName B 2024-11-08T18:40:20,720 DEBUG [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:20,720 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(327): Store=2db46d8a8e52bdcd84049ec79d0d81b7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:20,720 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,721 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2db46d8a8e52bdcd84049ec79d0d81b7 columnFamilyName C 2024-11-08T18:40:20,721 DEBUG [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:20,722 INFO [StoreOpener-2db46d8a8e52bdcd84049ec79d0d81b7-1 {}] regionserver.HStore(327): Store=2db46d8a8e52bdcd84049ec79d0d81b7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:20,722 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,722 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,723 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,724 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,725 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,725 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:40:20,726 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,727 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 2db46d8a8e52bdcd84049ec79d0d81b7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59313028, jitterRate=-0.11616700887680054}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:40:20,728 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,729 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 2db46d8a8e52bdcd84049ec79d0d81b7: Running coprocessor pre-open hook at 1731091220707Writing region info on filesystem at 1731091220707Initializing all the Stores at 1731091220708 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091220708Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091220708Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091220708Cleaning up temporary data from old regions at 1731091220725 (+17 ms)Running coprocessor post-open hooks at 1731091220728 (+3 ms)Region opened successfully at 1731091220729 (+1 ms) 2024-11-08T18:40:20,730 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., pid=39, masterSystemTime=1731091220703 2024-11-08T18:40:20,732 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,732 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,733 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=2db46d8a8e52bdcd84049ec79d0d81b7, regionState=OPEN, openSeqNum=5, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:20,735 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:40:20,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-08T18:40:20,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 in 185 msec 2024-11-08T18:40:20,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-08T18:40:20,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, REOPEN/MOVE in 510 msec 2024-11-08T18:40:20,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-08T18:40:20,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 520 msec 2024-11-08T18:40:20,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 946 msec 2024-11-08T18:40:20,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-08T18:40:20,750 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:20,751 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb0fd71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,751 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,751 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,752 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,752 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3654db63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,753 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,755 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f07f36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68ed5d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,757 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3035364d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,757 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,758 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c7f8a22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,760 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4406a4b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,760 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,760 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,761 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33f27bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,762 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8809974, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7230d245, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,765 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1931fafe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,766 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e881ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59b236c2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,768 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@542052e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,769 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,769 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53540, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,770 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c0cbfa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,771 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,771 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,771 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,772 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@657c0aa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,773 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754314f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58ae3464, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,775 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,775 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,776 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d933378, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,777 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,778 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21253c62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,779 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bbb11d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,779 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,779 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,780 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67fa0b36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,780 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,781 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,781 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,782 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cdff758, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,783 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d2a6fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,783 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:20,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:20,784 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cf83712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:20,786 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53618, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:20,786 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb02826, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:20,787 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,787 DEBUG [Thread-645 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,787 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,787 DEBUG [Thread-647 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,787 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,788 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:20,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,789 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,791 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,791 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-08T18:40:20,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,791 DEBUG [Thread-654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,792 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:20,792 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,792 DEBUG [Thread-643 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:20,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,793 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T18:40:20,793 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,794 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,794 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:20,794 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:20,795 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:20,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54148, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,799 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54162, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,803 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:40:20,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,820 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:20,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:40:20,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091280877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:20,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108060cc887043b42bcba8c135ceb212bae_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091220820/Put/seqid=0 2024-11-08T18:40:20,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091280881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:20,896 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:20,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091280882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:20,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091280883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091280884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:20,898 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T18:40:20,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741951_1127 (size=14594) 2024-11-08T18:40:20,903 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:20,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108060cc887043b42bcba8c135ceb212bae_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108060cc887043b42bcba8c135ceb212bae_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:20,920 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/2f020a7b8b974d3d87d7ea3e6e136f5c, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:20,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/2f020a7b8b974d3d87d7ea3e6e136f5c is 175, key is test_row_0/A:col10/1731091220820/Put/seqid=0 2024-11-08T18:40:20,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741952_1128 (size=39549) 2024-11-08T18:40:20,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T18:40:20,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:20,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:20,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:20,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:20,954 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/2f020a7b8b974d3d87d7ea3e6e136f5c 2024-11-08T18:40:20,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:20,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/92d496234918469d9399921782b88f54 is 50, key is test_row_0/B:col10/1731091220820/Put/seqid=0 2024-11-08T18:40:20,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:20,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091280990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091281000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091281000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,002 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091281009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091281010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741953_1129 (size=12001) 2024-11-08T18:40:21,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/92d496234918469d9399921782b88f54 2024-11-08T18:40:21,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/0356aed8c82d43a0b654ff5f8ed0bc1c is 50, key is test_row_0/C:col10/1731091220820/Put/seqid=0 2024-11-08T18:40:21,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T18:40:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:21,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741954_1130 (size=12001) 2024-11-08T18:40:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T18:40:21,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/0356aed8c82d43a0b654ff5f8ed0bc1c 2024-11-08T18:40:21,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/2f020a7b8b974d3d87d7ea3e6e136f5c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/2f020a7b8b974d3d87d7ea3e6e136f5c 2024-11-08T18:40:21,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/2f020a7b8b974d3d87d7ea3e6e136f5c, entries=200, sequenceid=17, filesize=38.6 K 2024-11-08T18:40:21,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/92d496234918469d9399921782b88f54 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/92d496234918469d9399921782b88f54 2024-11-08T18:40:21,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/92d496234918469d9399921782b88f54, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T18:40:21,159 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/0356aed8c82d43a0b654ff5f8ed0bc1c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/0356aed8c82d43a0b654ff5f8ed0bc1c 2024-11-08T18:40:21,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/0356aed8c82d43a0b654ff5f8ed0bc1c, entries=150, sequenceid=17, filesize=11.7 K 2024-11-08T18:40:21,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 347ms, sequenceid=17, compaction requested=false 2024-11-08T18:40:21,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:21,218 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:21,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:40:21,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108acb12ff7e1454406ac5820b0ac401bf6_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091220879/Put/seqid=0 2024-11-08T18:40:21,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T18:40:21,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:21,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:21,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:21,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,273 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091281246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,274 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091281254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091281259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,275 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,276 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741955_1131 (size=14594) 2024-11-08T18:40:21,285 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091281274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,290 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108acb12ff7e1454406ac5820b0ac401bf6_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108acb12ff7e1454406ac5820b0ac401bf6_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,292 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/b3d8e6541ea445cbbe1ba941ec6e6bac, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:21,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/b3d8e6541ea445cbbe1ba941ec6e6bac is 175, key is test_row_0/A:col10/1731091220879/Put/seqid=0 2024-11-08T18:40:21,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741956_1132 (size=39549) 2024-11-08T18:40:21,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091281277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,303 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/b3d8e6541ea445cbbe1ba941ec6e6bac 2024-11-08T18:40:21,337 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/794b021f764544d1bd2b8f10db2f81a1 is 50, key is test_row_0/B:col10/1731091220879/Put/seqid=0 2024-11-08T18:40:21,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741957_1133 (size=12001) 2024-11-08T18:40:21,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/794b021f764544d1bd2b8f10db2f81a1 2024-11-08T18:40:21,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091281380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091281381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,392 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091281383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/91865a7796ae4612be7d59ac1019def5 is 50, key is test_row_0/C:col10/1731091220879/Put/seqid=0 2024-11-08T18:40:21,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091281403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T18:40:21,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:21,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:21,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:21,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:21,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T18:40:21,421 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091281410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741958_1134 (size=12001) 2024-11-08T18:40:21,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/91865a7796ae4612be7d59ac1019def5 2024-11-08T18:40:21,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/b3d8e6541ea445cbbe1ba941ec6e6bac as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/b3d8e6541ea445cbbe1ba941ec6e6bac 2024-11-08T18:40:21,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/b3d8e6541ea445cbbe1ba941ec6e6bac, entries=200, sequenceid=41, filesize=38.6 K 2024-11-08T18:40:21,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/794b021f764544d1bd2b8f10db2f81a1 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/794b021f764544d1bd2b8f10db2f81a1 2024-11-08T18:40:21,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/794b021f764544d1bd2b8f10db2f81a1, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T18:40:21,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/91865a7796ae4612be7d59ac1019def5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/91865a7796ae4612be7d59ac1019def5 2024-11-08T18:40:21,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/91865a7796ae4612be7d59ac1019def5, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T18:40:21,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 244ms, sequenceid=41, compaction requested=false 2024-11-08T18:40:21,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:21,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-08T18:40:21,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:21,571 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:40:21,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083f7efbd522624da2be75282a37c07617_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091221272/Put/seqid=0 2024-11-08T18:40:21,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741959_1135 (size=12154) 2024-11-08T18:40:21,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:21,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:21,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:21,611 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083f7efbd522624da2be75282a37c07617_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083f7efbd522624da2be75282a37c07617_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:21,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/997ff78104b042c5928732337d8d35c2, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:21,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/997ff78104b042c5928732337d8d35c2 is 175, key is test_row_0/A:col10/1731091221272/Put/seqid=0 2024-11-08T18:40:21,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741960_1136 (size=30955) 2024-11-08T18:40:21,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091281669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,675 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091281671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091281671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091281673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091281675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091281781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091281789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091281789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091281789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091281790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:21,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T18:40:21,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:21,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091281989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,995 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:21,996 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,001 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091282000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091282000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,002 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091282010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091282010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,031 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=54, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/997ff78104b042c5928732337d8d35c2 2024-11-08T18:40:22,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/7d11f12316464feab7dea8d65e8aa2c5 is 50, key is test_row_0/B:col10/1731091221272/Put/seqid=0 2024-11-08T18:40:22,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741961_1137 (size=12001) 2024-11-08T18:40:22,092 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/7d11f12316464feab7dea8d65e8aa2c5 2024-11-08T18:40:22,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/8ba6d2a5370a4dd5bf183cd44c93fe46 is 50, key is test_row_0/C:col10/1731091221272/Put/seqid=0 2024-11-08T18:40:22,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741962_1138 (size=12001) 2024-11-08T18:40:22,144 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/8ba6d2a5370a4dd5bf183cd44c93fe46 2024-11-08T18:40:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/997ff78104b042c5928732337d8d35c2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/997ff78104b042c5928732337d8d35c2 2024-11-08T18:40:22,166 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/997ff78104b042c5928732337d8d35c2, entries=150, sequenceid=54, filesize=30.2 K 2024-11-08T18:40:22,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/7d11f12316464feab7dea8d65e8aa2c5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7d11f12316464feab7dea8d65e8aa2c5 2024-11-08T18:40:22,174 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7d11f12316464feab7dea8d65e8aa2c5, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T18:40:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/8ba6d2a5370a4dd5bf183cd44c93fe46 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8ba6d2a5370a4dd5bf183cd44c93fe46 2024-11-08T18:40:22,187 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8ba6d2a5370a4dd5bf183cd44c93fe46, entries=150, sequenceid=54, filesize=11.7 K 2024-11-08T18:40:22,190 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 618ms, sequenceid=54, compaction requested=true 2024-11-08T18:40:22,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:22,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:22,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-08T18:40:22,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-08T18:40:22,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-08T18:40:22,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4010 sec 2024-11-08T18:40:22,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.4150 sec 2024-11-08T18:40:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:22,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:22,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:40:22,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091282320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,329 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081a822233e8d54aefbe6b9bf085bf4335_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091221672/Put/seqid=0 2024-11-08T18:40:22,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091282325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091282326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,335 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,335 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091282328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091282328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741963_1139 (size=14594) 2024-11-08T18:40:22,357 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,362 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081a822233e8d54aefbe6b9bf085bf4335_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a822233e8d54aefbe6b9bf085bf4335_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:22,366 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/acb0c5bc81c44aa693250d655e094421, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:22,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/acb0c5bc81c44aa693250d655e094421 is 175, key is test_row_0/A:col10/1731091221672/Put/seqid=0 2024-11-08T18:40:22,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741964_1140 (size=39549) 2024-11-08T18:40:22,381 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/acb0c5bc81c44aa693250d655e094421 2024-11-08T18:40:22,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/1c66dff60b5041b9bf40fa574fef95b4 is 50, key is test_row_0/B:col10/1731091221672/Put/seqid=0 2024-11-08T18:40:22,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741965_1141 (size=12001) 2024-11-08T18:40:22,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091282440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091282440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091282440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091282440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,446 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091282650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091282650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,653 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091282651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,654 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091282660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/1c66dff60b5041b9bf40fa574fef95b4 2024-11-08T18:40:22,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4539076bf2a346f4aa7ae19f0f676a20 is 50, key is test_row_0/C:col10/1731091221672/Put/seqid=0 2024-11-08T18:40:22,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:22,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091282840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:22,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:22,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741966_1142 (size=12001) 2024-11-08T18:40:22,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4539076bf2a346f4aa7ae19f0f676a20 2024-11-08T18:40:22,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/acb0c5bc81c44aa693250d655e094421 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/acb0c5bc81c44aa693250d655e094421 2024-11-08T18:40:22,869 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/acb0c5bc81c44aa693250d655e094421, entries=200, sequenceid=78, filesize=38.6 K 2024-11-08T18:40:22,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/1c66dff60b5041b9bf40fa574fef95b4 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1c66dff60b5041b9bf40fa574fef95b4 2024-11-08T18:40:22,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1c66dff60b5041b9bf40fa574fef95b4, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T18:40:22,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4539076bf2a346f4aa7ae19f0f676a20 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4539076bf2a346f4aa7ae19f0f676a20 2024-11-08T18:40:22,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4539076bf2a346f4aa7ae19f0f676a20, entries=150, sequenceid=78, filesize=11.7 K 2024-11-08T18:40:22,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 603ms, sequenceid=78, compaction requested=true 2024-11-08T18:40:22,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:22,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,906 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:22,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:22,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:22,907 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:22,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:22,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:22,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:22,910 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149602 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:22,910 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:22,910 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/A is initiating minor compaction (all files) 2024-11-08T18:40:22,910 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/A in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:22,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,910 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/2f020a7b8b974d3d87d7ea3e6e136f5c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/b3d8e6541ea445cbbe1ba941ec6e6bac, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/997ff78104b042c5928732337d8d35c2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/acb0c5bc81c44aa693250d655e094421] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=146.1 K 2024-11-08T18:40:22,910 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:22,911 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/2f020a7b8b974d3d87d7ea3e6e136f5c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/b3d8e6541ea445cbbe1ba941ec6e6bac, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/997ff78104b042c5928732337d8d35c2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/acb0c5bc81c44aa693250d655e094421] 2024-11-08T18:40:22,911 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:22,911 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f020a7b8b974d3d87d7ea3e6e136f5c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731091220803 2024-11-08T18:40:22,911 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/B is initiating minor compaction (all files) 2024-11-08T18:40:22,911 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/B in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:22,911 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/92d496234918469d9399921782b88f54, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/794b021f764544d1bd2b8f10db2f81a1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7d11f12316464feab7dea8d65e8aa2c5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1c66dff60b5041b9bf40fa574fef95b4] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=46.9 K 2024-11-08T18:40:22,912 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3d8e6541ea445cbbe1ba941ec6e6bac, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731091220870 2024-11-08T18:40:22,912 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 997ff78104b042c5928732337d8d35c2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731091221221 2024-11-08T18:40:22,912 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 92d496234918469d9399921782b88f54, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731091220816 2024-11-08T18:40:22,912 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting acb0c5bc81c44aa693250d655e094421, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731091221652 2024-11-08T18:40:22,913 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 794b021f764544d1bd2b8f10db2f81a1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731091220870 2024-11-08T18:40:22,913 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d11f12316464feab7dea8d65e8aa2c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731091221221 2024-11-08T18:40:22,914 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c66dff60b5041b9bf40fa574fef95b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731091221652 2024-11-08T18:40:22,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,930 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:22,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,942 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108b14db17a9f1d411084408214691e4da1_2db46d8a8e52bdcd84049ec79d0d81b7 store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:22,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-08T18:40:22,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,956 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,959 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#B#compaction#124 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:22,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,959 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/7f37d2dd439447a3adacd989b92b30b6 is 50, key is test_row_0/B:col10/1731091221672/Put/seqid=0 2024-11-08T18:40:22,960 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:22,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-08T18:40:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T18:40:22,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,963 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:22,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,964 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:22,964 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,966 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108b14db17a9f1d411084408214691e4da1_2db46d8a8e52bdcd84049ec79d0d81b7, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:22,966 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b14db17a9f1d411084408214691e4da1_2db46d8a8e52bdcd84049ec79d0d81b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:22,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:22,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741967_1143 (size=4469) 2024-11-08T18:40:22,995 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#A#compaction#123 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:22,995 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/c93d1ec9ef844d05b6dd469a3708e2c3 is 175, key is test_row_0/A:col10/1731091221672/Put/seqid=0 2024-11-08T18:40:23,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741969_1145 (size=31093) 2024-11-08T18:40:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,014 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/c93d1ec9ef844d05b6dd469a3708e2c3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/c93d1ec9ef844d05b6dd469a3708e2c3 2024-11-08T18:40:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741968_1144 (size=12139) 2024-11-08T18:40:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,026 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/A of 2db46d8a8e52bdcd84049ec79d0d81b7 into c93d1ec9ef844d05b6dd469a3708e2c3(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:23,026 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:23,026 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/A, priority=12, startTime=1731091222906; duration=0sec 2024-11-08T18:40:23,026 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:23,026 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:A 2024-11-08T18:40:23,026 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,028 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:23,028 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/C is initiating minor compaction (all files) 2024-11-08T18:40:23,028 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/C in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:23,028 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/0356aed8c82d43a0b654ff5f8ed0bc1c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/91865a7796ae4612be7d59ac1019def5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8ba6d2a5370a4dd5bf183cd44c93fe46, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4539076bf2a346f4aa7ae19f0f676a20] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=46.9 K 2024-11-08T18:40:23,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,029 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0356aed8c82d43a0b654ff5f8ed0bc1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731091220816 2024-11-08T18:40:23,029 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91865a7796ae4612be7d59ac1019def5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731091220870 2024-11-08T18:40:23,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,030 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ba6d2a5370a4dd5bf183cd44c93fe46, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1731091221221 2024-11-08T18:40:23,030 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4539076bf2a346f4aa7ae19f0f676a20, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731091221652 2024-11-08T18:40:23,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:23,045 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:23,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:40:23,057 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#C#compaction#125 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:23,058 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/b2668ef4c99f4475b02896714d280c98 is 50, key is test_row_0/C:col10/1731091221672/Put/seqid=0 2024-11-08T18:40:23,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T18:40:23,092 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083b97dc004c57439b879474b47b63d82d_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091223046/Put/seqid=0 2024-11-08T18:40:23,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741970_1146 (size=12139) 2024-11-08T18:40:23,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T18:40:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:23,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:23,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:23,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:23,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741971_1147 (size=17034) 2024-11-08T18:40:23,129 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,135 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083b97dc004c57439b879474b47b63d82d_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b97dc004c57439b879474b47b63d82d_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:23,137 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/3dd2f8d9745c48099f6abcd06ef53cc5, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:23,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/3dd2f8d9745c48099f6abcd06ef53cc5 is 175, key is test_row_0/A:col10/1731091223046/Put/seqid=0 2024-11-08T18:40:23,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741972_1148 (size=48139) 2024-11-08T18:40:23,157 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/3dd2f8d9745c48099f6abcd06ef53cc5 2024-11-08T18:40:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091283149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091283154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091283158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,162 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091283159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,164 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/23e5d26d69fc408286b5d318e4661c7a is 50, key is test_row_0/B:col10/1731091223046/Put/seqid=0 2024-11-08T18:40:23,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741973_1149 (size=12001) 2024-11-08T18:40:23,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/23e5d26d69fc408286b5d318e4661c7a 2024-11-08T18:40:23,244 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/1722185e25774cb4b917d13eb7c703d2 is 50, key is test_row_0/C:col10/1731091223046/Put/seqid=0 2024-11-08T18:40:23,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741974_1150 (size=12001) 2024-11-08T18:40:23,263 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/1722185e25774cb4b917d13eb7c703d2 2024-11-08T18:40:23,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T18:40:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:23,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:23,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:23,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:23,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:23,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T18:40:23,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/3dd2f8d9745c48099f6abcd06ef53cc5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3dd2f8d9745c48099f6abcd06ef53cc5 2024-11-08T18:40:23,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091283272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,287 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091283273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091283273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,290 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091283273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3dd2f8d9745c48099f6abcd06ef53cc5, entries=250, sequenceid=94, filesize=47.0 K 2024-11-08T18:40:23,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/23e5d26d69fc408286b5d318e4661c7a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/23e5d26d69fc408286b5d318e4661c7a 2024-11-08T18:40:23,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/23e5d26d69fc408286b5d318e4661c7a, entries=150, sequenceid=94, filesize=11.7 K 2024-11-08T18:40:23,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/1722185e25774cb4b917d13eb7c703d2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/1722185e25774cb4b917d13eb7c703d2 2024-11-08T18:40:23,324 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/1722185e25774cb4b917d13eb7c703d2, entries=150, sequenceid=94, filesize=11.7 K 2024-11-08T18:40:23,327 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 281ms, sequenceid=94, compaction requested=false 2024-11-08T18:40:23,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:23,428 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-08T18:40:23,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:23,429 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T18:40:23,434 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/7f37d2dd439447a3adacd989b92b30b6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7f37d2dd439447a3adacd989b92b30b6 2024-11-08T18:40:23,450 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/B of 2db46d8a8e52bdcd84049ec79d0d81b7 into 7f37d2dd439447a3adacd989b92b30b6(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:23,450 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:23,450 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/B, priority=12, startTime=1731091222907; duration=0sec 2024-11-08T18:40:23,450 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:23,450 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:B 2024-11-08T18:40:23,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110845d3173727c54275b4e53b2817bb8bda_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091223066/Put/seqid=0 2024-11-08T18:40:23,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741975_1151 (size=12154) 2024-11-08T18:40:23,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:23,492 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:23,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:23,496 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110845d3173727c54275b4e53b2817bb8bda_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845d3173727c54275b4e53b2817bb8bda_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:23,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/154ecd7bc76440deae4679f786ef362f, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:23,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/154ecd7bc76440deae4679f786ef362f is 175, key is test_row_0/A:col10/1731091223066/Put/seqid=0 2024-11-08T18:40:23,516 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/b2668ef4c99f4475b02896714d280c98 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b2668ef4c99f4475b02896714d280c98 2024-11-08T18:40:23,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741976_1152 (size=30955) 2024-11-08T18:40:23,531 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/C of 2db46d8a8e52bdcd84049ec79d0d81b7 into b2668ef4c99f4475b02896714d280c98(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:23,532 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:23,532 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/C, priority=12, startTime=1731091222907; duration=0sec 2024-11-08T18:40:23,532 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:23,532 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:C 2024-11-08T18:40:23,544 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T18:40:23,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091283534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091283536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,548 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091283546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091283547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T18:40:23,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091283650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,667 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091283660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,669 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091283660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,672 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,673 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091283660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091283859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091283859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,865 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,866 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,867 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091283879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091283879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,883 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091283890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:23,921 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/154ecd7bc76440deae4679f786ef362f 2024-11-08T18:40:23,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/3b59826514394186ad28369245de9934 is 50, key is test_row_0/B:col10/1731091223066/Put/seqid=0 2024-11-08T18:40:23,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741977_1153 (size=12001) 2024-11-08T18:40:23,987 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/3b59826514394186ad28369245de9934 2024-11-08T18:40:24,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b is 50, key is test_row_0/C:col10/1731091223066/Put/seqid=0 2024-11-08T18:40:24,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741978_1154 (size=12001) 2024-11-08T18:40:24,035 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b 2024-11-08T18:40:24,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/154ecd7bc76440deae4679f786ef362f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/154ecd7bc76440deae4679f786ef362f 2024-11-08T18:40:24,053 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/154ecd7bc76440deae4679f786ef362f, entries=150, sequenceid=115, filesize=30.2 K 2024-11-08T18:40:24,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/3b59826514394186ad28369245de9934 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/3b59826514394186ad28369245de9934 2024-11-08T18:40:24,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/3b59826514394186ad28369245de9934, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T18:40:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b 2024-11-08T18:40:24,079 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T18:40:24,081 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 652ms, sequenceid=115, compaction requested=true 2024-11-08T18:40:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-08T18:40:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-08T18:40:24,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-08T18:40:24,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1190 sec 2024-11-08T18:40:24,091 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.1270 sec 2024-11-08T18:40:24,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-08T18:40:24,107 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:24,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-08T18:40:24,111 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:24,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T18:40:24,113 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:24,113 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:24,183 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:24,184 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T18:40:24,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:24,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fba102c1e83f4c7c99517445da4a327e_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091223532/Put/seqid=0 2024-11-08T18:40:24,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741979_1155 (size=14744) 2024-11-08T18:40:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T18:40:24,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,250 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091284241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091284239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,251 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,252 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091284250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,267 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:24,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:24,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091284258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091284360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091284360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,373 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091284369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,374 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091284380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,381 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T18:40:24,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091284579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091284579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091284580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091284590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,596 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,612 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:24,617 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108fba102c1e83f4c7c99517445da4a327e_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fba102c1e83f4c7c99517445da4a327e_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:24,619 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/fb0f056e69ab41c4938b81233964f480, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:24,620 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/fb0f056e69ab41c4938b81233964f480 is 175, key is test_row_0/A:col10/1731091223532/Put/seqid=0 2024-11-08T18:40:24,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741980_1156 (size=39699) 2024-11-08T18:40:24,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=133, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/fb0f056e69ab41c4938b81233964f480 2024-11-08T18:40:24,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/40dfb84bbe6047669b8eb6d75d42f1ba is 50, key is test_row_0/B:col10/1731091223532/Put/seqid=0 2024-11-08T18:40:24,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741981_1157 (size=12101) 2024-11-08T18:40:24,734 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:24,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:24,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T18:40:24,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091284890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,892 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091284890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,893 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:24,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:24,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:24,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:24,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091284895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,901 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:24,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091284900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:24,905 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:25,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,075 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/40dfb84bbe6047669b8eb6d75d42f1ba 2024-11-08T18:40:25,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/702b7a4ce0834888a792f33cea19abf0 is 50, key is test_row_0/C:col10/1731091223532/Put/seqid=0 2024-11-08T18:40:25,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741982_1158 (size=12101) 2024-11-08T18:40:25,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:25,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T18:40:25,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:25,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:25,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,360 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:25,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091285399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:25,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:25,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091285400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,402 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:25,419 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:25,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091285410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,420 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:25,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091285419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,425 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:25,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=133 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/702b7a4ce0834888a792f33cea19abf0 2024-11-08T18:40:25,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/fb0f056e69ab41c4938b81233964f480 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/fb0f056e69ab41c4938b81233964f480 2024-11-08T18:40:25,512 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/fb0f056e69ab41c4938b81233964f480, entries=200, sequenceid=133, filesize=38.8 K 2024-11-08T18:40:25,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/40dfb84bbe6047669b8eb6d75d42f1ba as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/40dfb84bbe6047669b8eb6d75d42f1ba 2024-11-08T18:40:25,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:25,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:25,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/40dfb84bbe6047669b8eb6d75d42f1ba, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T18:40:25,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/702b7a4ce0834888a792f33cea19abf0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/702b7a4ce0834888a792f33cea19abf0 2024-11-08T18:40:25,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/702b7a4ce0834888a792f33cea19abf0, entries=150, sequenceid=133, filesize=11.8 K 2024-11-08T18:40:25,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1379ms, sequenceid=133, compaction requested=true 2024-11-08T18:40:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:25,563 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:25,563 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-08T18:40:25,563 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:25,567 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:25,567 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/B is initiating minor compaction (all files) 2024-11-08T18:40:25,568 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/B in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,568 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7f37d2dd439447a3adacd989b92b30b6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/23e5d26d69fc408286b5d318e4661c7a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/3b59826514394186ad28369245de9934, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/40dfb84bbe6047669b8eb6d75d42f1ba] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=47.1 K 2024-11-08T18:40:25,568 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:25,568 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/A is initiating minor compaction (all files) 2024-11-08T18:40:25,568 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/A in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,568 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/c93d1ec9ef844d05b6dd469a3708e2c3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3dd2f8d9745c48099f6abcd06ef53cc5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/154ecd7bc76440deae4679f786ef362f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/fb0f056e69ab41c4938b81233964f480] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=146.4 K 2024-11-08T18:40:25,568 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,568 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/c93d1ec9ef844d05b6dd469a3708e2c3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3dd2f8d9745c48099f6abcd06ef53cc5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/154ecd7bc76440deae4679f786ef362f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/fb0f056e69ab41c4938b81233964f480] 2024-11-08T18:40:25,569 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c93d1ec9ef844d05b6dd469a3708e2c3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731091221652 2024-11-08T18:40:25,569 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f37d2dd439447a3adacd989b92b30b6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731091221652 2024-11-08T18:40:25,570 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 23e5d26d69fc408286b5d318e4661c7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091222326 2024-11-08T18:40:25,570 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dd2f8d9745c48099f6abcd06ef53cc5, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091222326 2024-11-08T18:40:25,570 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b59826514394186ad28369245de9934, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731091223066 2024-11-08T18:40:25,571 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 154ecd7bc76440deae4679f786ef362f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731091223066 2024-11-08T18:40:25,571 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 40dfb84bbe6047669b8eb6d75d42f1ba, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091223512 2024-11-08T18:40:25,572 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fb0f056e69ab41c4938b81233964f480, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091223512 2024-11-08T18:40:25,597 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#B#compaction#135 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:25,598 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/1dcd8ceedeac4eb0948125e0f88b1f83 is 50, key is test_row_0/B:col10/1731091223532/Put/seqid=0 2024-11-08T18:40:25,603 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:25,606 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411082ce9e76e90b34dc5ac1aba0e8c243f76_2db46d8a8e52bdcd84049ec79d0d81b7 store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:25,610 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411082ce9e76e90b34dc5ac1aba0e8c243f76_2db46d8a8e52bdcd84049ec79d0d81b7, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:25,610 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082ce9e76e90b34dc5ac1aba0e8c243f76_2db46d8a8e52bdcd84049ec79d0d81b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:25,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741983_1159 (size=12375) 2024-11-08T18:40:25,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741984_1160 (size=4469) 2024-11-08T18:40:25,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-08T18:40:25,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:25,676 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T18:40:25,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d881574c24af4352873da7de982b141b_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091224219/Put/seqid=0 2024-11-08T18:40:25,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741985_1161 (size=12304) 2024-11-08T18:40:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:25,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:25,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091285975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:25,984 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:26,058 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#A#compaction#136 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:26,059 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/7f97ae8ded484badb49b5d622cb6b49c is 175, key is test_row_0/A:col10/1731091223532/Put/seqid=0 2024-11-08T18:40:26,070 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/1dcd8ceedeac4eb0948125e0f88b1f83 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1dcd8ceedeac4eb0948125e0f88b1f83 2024-11-08T18:40:26,081 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/B of 2db46d8a8e52bdcd84049ec79d0d81b7 into 1dcd8ceedeac4eb0948125e0f88b1f83(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:26,081 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:26,081 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/B, priority=12, startTime=1731091225563; duration=0sec 2024-11-08T18:40:26,082 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:26,082 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:B 2024-11-08T18:40:26,082 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:26,085 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:26,086 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/C is initiating minor compaction (all files) 2024-11-08T18:40:26,086 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/C in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:26,086 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b2668ef4c99f4475b02896714d280c98, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/1722185e25774cb4b917d13eb7c703d2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/702b7a4ce0834888a792f33cea19abf0] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=47.1 K 2024-11-08T18:40:26,087 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting b2668ef4c99f4475b02896714d280c98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1731091221652 2024-11-08T18:40:26,087 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1722185e25774cb4b917d13eb7c703d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091222326 2024-11-08T18:40:26,088 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d891e2ff1dbb4ec1a1eee9f7d7c81c6b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731091223066 2024-11-08T18:40:26,095 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 702b7a4ce0834888a792f33cea19abf0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091223512 2024-11-08T18:40:26,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741986_1162 (size=31329) 2024-11-08T18:40:26,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091286094, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,099 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:26,110 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/7f97ae8ded484badb49b5d622cb6b49c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7f97ae8ded484badb49b5d622cb6b49c 2024-11-08T18:40:26,118 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/A of 2db46d8a8e52bdcd84049ec79d0d81b7 into 7f97ae8ded484badb49b5d622cb6b49c(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:26,118 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:26,118 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/A, priority=12, startTime=1731091225563; duration=0sec 2024-11-08T18:40:26,119 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:26,119 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:A 2024-11-08T18:40:26,142 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#C#compaction#138 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:26,142 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/ad531aca7b8340f79083d27c81e30808 is 50, key is test_row_0/C:col10/1731091223532/Put/seqid=0 2024-11-08T18:40:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:26,159 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d881574c24af4352873da7de982b141b_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d881574c24af4352873da7de982b141b_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/aa108946d50e46ab9cd2db6510554919, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:26,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/aa108946d50e46ab9cd2db6510554919 is 175, key is test_row_0/A:col10/1731091224219/Put/seqid=0 2024-11-08T18:40:26,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741987_1163 (size=12375) 2024-11-08T18:40:26,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741988_1164 (size=31105) 2024-11-08T18:40:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T18:40:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091286310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:26,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091286409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:26,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091286409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:26,412 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,412 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:26,413 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,413 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091286430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:26,434 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:26,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091286439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:26,440 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,441 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:26,602 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/ad531aca7b8340f79083d27c81e30808 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ad531aca7b8340f79083d27c81e30808 2024-11-08T18:40:26,610 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/C of 2db46d8a8e52bdcd84049ec79d0d81b7 into ad531aca7b8340f79083d27c81e30808(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:26,610 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:26,610 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/C, priority=12, startTime=1731091225563; duration=0sec 2024-11-08T18:40:26,610 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:26,610 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:C 2024-11-08T18:40:26,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:26,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091286619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:26,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:26,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/aa108946d50e46ab9cd2db6510554919 2024-11-08T18:40:26,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/bcbcf5cca0c7473887d38d2ee26dacc2 is 50, key is test_row_0/B:col10/1731091224219/Put/seqid=0 2024-11-08T18:40:26,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741989_1165 (size=12151) 2024-11-08T18:40:26,662 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/bcbcf5cca0c7473887d38d2ee26dacc2 2024-11-08T18:40:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4c43f3d01eac4adb8c026ab265752a82 is 50, key is test_row_0/C:col10/1731091224219/Put/seqid=0 2024-11-08T18:40:26,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741990_1166 (size=12151) 2024-11-08T18:40:26,700 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4c43f3d01eac4adb8c026ab265752a82 2024-11-08T18:40:26,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/aa108946d50e46ab9cd2db6510554919 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/aa108946d50e46ab9cd2db6510554919 2024-11-08T18:40:26,725 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/aa108946d50e46ab9cd2db6510554919, entries=150, sequenceid=153, filesize=30.4 K 2024-11-08T18:40:26,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/bcbcf5cca0c7473887d38d2ee26dacc2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/bcbcf5cca0c7473887d38d2ee26dacc2 2024-11-08T18:40:26,734 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/bcbcf5cca0c7473887d38d2ee26dacc2, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T18:40:26,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4c43f3d01eac4adb8c026ab265752a82 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4c43f3d01eac4adb8c026ab265752a82 2024-11-08T18:40:26,742 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4c43f3d01eac4adb8c026ab265752a82, entries=150, sequenceid=153, filesize=11.9 K 2024-11-08T18:40:26,743 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1067ms, sequenceid=153, compaction requested=false 2024-11-08T18:40:26,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:26,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:26,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-08T18:40:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-08T18:40:26,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-08T18:40:26,747 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-11-08T18:40:26,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 2.6390 sec 2024-11-08T18:40:27,126 INFO [master/b75fe890c219:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-08T18:40:27,127 INFO [master/b75fe890c219:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-08T18:40:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:27,134 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:27,134 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T18:40:27,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110821ec513486704763b3858bdcc444659d_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091225965/Put/seqid=0 2024-11-08T18:40:27,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741991_1167 (size=12304) 2024-11-08T18:40:27,223 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:27,234 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110821ec513486704763b3858bdcc444659d_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110821ec513486704763b3858bdcc444659d_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:27,235 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/dca55cf0d9aa4c7a885aa91552fca18a, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:27,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/dca55cf0d9aa4c7a885aa91552fca18a is 175, key is test_row_0/A:col10/1731091225965/Put/seqid=0 2024-11-08T18:40:27,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741992_1168 (size=31105) 2024-11-08T18:40:27,246 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/dca55cf0d9aa4c7a885aa91552fca18a 2024-11-08T18:40:27,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/16146265cbc542e3abc29a4beb240949 is 50, key is test_row_0/B:col10/1731091225965/Put/seqid=0 2024-11-08T18:40:27,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091287265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:27,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:27,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741993_1169 (size=12151) 2024-11-08T18:40:27,281 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/16146265cbc542e3abc29a4beb240949 2024-11-08T18:40:27,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/dfa00026b2c1438b9c92ef96b4495502 is 50, key is test_row_0/C:col10/1731091225965/Put/seqid=0 2024-11-08T18:40:27,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741994_1170 (size=12151) 2024-11-08T18:40:27,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091287379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:27,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:27,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091287590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:27,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:27,712 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/dfa00026b2c1438b9c92ef96b4495502 2024-11-08T18:40:27,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/dca55cf0d9aa4c7a885aa91552fca18a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dca55cf0d9aa4c7a885aa91552fca18a 2024-11-08T18:40:27,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dca55cf0d9aa4c7a885aa91552fca18a, entries=150, sequenceid=173, filesize=30.4 K 2024-11-08T18:40:27,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/16146265cbc542e3abc29a4beb240949 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/16146265cbc542e3abc29a4beb240949 2024-11-08T18:40:27,749 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/16146265cbc542e3abc29a4beb240949, entries=150, sequenceid=173, filesize=11.9 K 2024-11-08T18:40:27,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/dfa00026b2c1438b9c92ef96b4495502 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/dfa00026b2c1438b9c92ef96b4495502 2024-11-08T18:40:27,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/dfa00026b2c1438b9c92ef96b4495502, entries=150, sequenceid=173, filesize=11.9 K 2024-11-08T18:40:27,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 627ms, sequenceid=173, compaction requested=true 2024-11-08T18:40:27,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:27,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:27,762 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:27,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:27,762 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:27,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:27,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:27,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:27,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:27,764 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:27,764 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/B is initiating minor compaction (all files) 2024-11-08T18:40:27,764 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/B in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:27,764 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1dcd8ceedeac4eb0948125e0f88b1f83, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/bcbcf5cca0c7473887d38d2ee26dacc2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/16146265cbc542e3abc29a4beb240949] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=35.8 K 2024-11-08T18:40:27,771 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93539 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:27,771 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dcd8ceedeac4eb0948125e0f88b1f83, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091223512 2024-11-08T18:40:27,771 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/A is initiating minor compaction (all files) 2024-11-08T18:40:27,771 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/A in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:27,771 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7f97ae8ded484badb49b5d622cb6b49c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/aa108946d50e46ab9cd2db6510554919, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dca55cf0d9aa4c7a885aa91552fca18a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=91.3 K 2024-11-08T18:40:27,771 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting bcbcf5cca0c7473887d38d2ee26dacc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731091224219 2024-11-08T18:40:27,771 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:27,771 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7f97ae8ded484badb49b5d622cb6b49c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/aa108946d50e46ab9cd2db6510554919, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dca55cf0d9aa4c7a885aa91552fca18a] 2024-11-08T18:40:27,772 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 16146265cbc542e3abc29a4beb240949, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731091225956 2024-11-08T18:40:27,772 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f97ae8ded484badb49b5d622cb6b49c, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091223512 2024-11-08T18:40:27,772 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa108946d50e46ab9cd2db6510554919, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731091224219 2024-11-08T18:40:27,773 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting dca55cf0d9aa4c7a885aa91552fca18a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731091225956 2024-11-08T18:40:27,801 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#B#compaction#144 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:27,801 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/15ff038fb9aa4d9da558c26620b08352 is 50, key is test_row_0/B:col10/1731091225965/Put/seqid=0 2024-11-08T18:40:27,810 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:27,831 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108b660be1a6fba424587dc645d08928c9a_2db46d8a8e52bdcd84049ec79d0d81b7 store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:27,833 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108b660be1a6fba424587dc645d08928c9a_2db46d8a8e52bdcd84049ec79d0d81b7, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:27,834 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b660be1a6fba424587dc645d08928c9a_2db46d8a8e52bdcd84049ec79d0d81b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:27,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741995_1171 (size=12527) 2024-11-08T18:40:27,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741996_1172 (size=4469) 2024-11-08T18:40:27,852 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#A#compaction#145 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:27,852 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/3666808d895846dba46f8f3d332ed200 is 175, key is test_row_0/A:col10/1731091225965/Put/seqid=0 2024-11-08T18:40:27,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741997_1173 (size=31481) 2024-11-08T18:40:27,872 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/3666808d895846dba46f8f3d332ed200 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3666808d895846dba46f8f3d332ed200 2024-11-08T18:40:27,885 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/A of 2db46d8a8e52bdcd84049ec79d0d81b7 into 3666808d895846dba46f8f3d332ed200(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:27,885 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:27,886 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/A, priority=13, startTime=1731091227762; duration=0sec 2024-11-08T18:40:27,886 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:27,886 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:A 2024-11-08T18:40:27,886 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:27,889 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:27,889 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/C is initiating minor compaction (all files) 2024-11-08T18:40:27,889 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/C in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:27,889 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ad531aca7b8340f79083d27c81e30808, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4c43f3d01eac4adb8c026ab265752a82, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/dfa00026b2c1438b9c92ef96b4495502] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=35.8 K 2024-11-08T18:40:27,890 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad531aca7b8340f79083d27c81e30808, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=133, earliestPutTs=1731091223512 2024-11-08T18:40:27,891 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c43f3d01eac4adb8c026ab265752a82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731091224219 2024-11-08T18:40:27,891 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfa00026b2c1438b9c92ef96b4495502, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731091225956 2024-11-08T18:40:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:27,905 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:27,905 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-08T18:40:27,908 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#C#compaction#146 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:27,909 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/78d650118e64451f858ee217bf673ebf is 50, key is test_row_0/C:col10/1731091225965/Put/seqid=0 2024-11-08T18:40:27,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741998_1174 (size=12527) 2024-11-08T18:40:27,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081e86e886c38240e8bf69467613695c3d_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091227903/Put/seqid=0 2024-11-08T18:40:27,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741999_1175 (size=14794) 2024-11-08T18:40:27,957 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:27,971 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081e86e886c38240e8bf69467613695c3d_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081e86e886c38240e8bf69467613695c3d_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:27,972 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/f9316d24bea04427a61b335095be3088, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:27,973 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/f9316d24bea04427a61b335095be3088 is 175, key is test_row_0/A:col10/1731091227903/Put/seqid=0 2024-11-08T18:40:27,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742000_1176 (size=39749) 2024-11-08T18:40:28,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:28,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091288000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:28,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:28,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091288109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:28,250 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/15ff038fb9aa4d9da558c26620b08352 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/15ff038fb9aa4d9da558c26620b08352 2024-11-08T18:40:28,262 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/B of 2db46d8a8e52bdcd84049ec79d0d81b7 into 15ff038fb9aa4d9da558c26620b08352(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:28,263 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:28,263 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/B, priority=13, startTime=1731091227762; duration=0sec 2024-11-08T18:40:28,263 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:28,263 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:B 2024-11-08T18:40:28,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-08T18:40:28,280 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:28,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:28,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-08T18:40:28,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T18:40:28,282 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:28,283 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:28,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:28,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:28,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091288319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:28,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:28,347 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/78d650118e64451f858ee217bf673ebf as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/78d650118e64451f858ee217bf673ebf 2024-11-08T18:40:28,363 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/C of 2db46d8a8e52bdcd84049ec79d0d81b7 into 78d650118e64451f858ee217bf673ebf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:28,363 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:28,363 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/C, priority=13, startTime=1731091227762; duration=0sec 2024-11-08T18:40:28,364 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:28,364 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:C 2024-11-08T18:40:28,381 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/f9316d24bea04427a61b335095be3088 2024-11-08T18:40:28,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T18:40:28,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/f200eac8d2cb474e8e399e8d6b7a1dc9 is 50, key is test_row_0/B:col10/1731091227903/Put/seqid=0 2024-11-08T18:40:28,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742001_1177 (size=12151) 2024-11-08T18:40:28,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/f200eac8d2cb474e8e399e8d6b7a1dc9 2024-11-08T18:40:28,429 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/8a1d65dd70484d458a41e2d0c7221aef is 50, key is test_row_0/C:col10/1731091227903/Put/seqid=0 2024-11-08T18:40:28,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:28,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091288430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:28,435 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,436 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:28,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T18:40:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:28,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091288440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:28,443 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:28,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:28,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091288449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:28,456 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,457 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:28,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091288459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:28,465 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:28,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742002_1178 (size=12151) 2024-11-08T18:40:28,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T18:40:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:28,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:28,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T18:40:28,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:28,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091288629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:28,630 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:28,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T18:40:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:28,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:28,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/8a1d65dd70484d458a41e2d0c7221aef 2024-11-08T18:40:28,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/f9316d24bea04427a61b335095be3088 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f9316d24bea04427a61b335095be3088 2024-11-08T18:40:28,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f9316d24bea04427a61b335095be3088, entries=200, sequenceid=194, filesize=38.8 K 2024-11-08T18:40:28,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/f200eac8d2cb474e8e399e8d6b7a1dc9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/f200eac8d2cb474e8e399e8d6b7a1dc9 2024-11-08T18:40:28,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/f200eac8d2cb474e8e399e8d6b7a1dc9, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T18:40:28,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/8a1d65dd70484d458a41e2d0c7221aef as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8a1d65dd70484d458a41e2d0c7221aef 2024-11-08T18:40:28,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T18:40:28,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:28,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:28,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:28,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:28,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8a1d65dd70484d458a41e2d0c7221aef, entries=150, sequenceid=194, filesize=11.9 K 2024-11-08T18:40:28,904 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 998ms, sequenceid=194, compaction requested=false 2024-11-08T18:40:28,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T18:40:29,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-08T18:40:29,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:29,051 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T18:40:29,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110800b2f5417bc6419593d2b14ee079aa44_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091227971/Put/seqid=0 2024-11-08T18:40:29,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742003_1179 (size=12304) 2024-11-08T18:40:29,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:29,088 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110800b2f5417bc6419593d2b14ee079aa44_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110800b2f5417bc6419593d2b14ee079aa44_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:29,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/9ed9549d03c84373a855824519c38aff, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:29,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/9ed9549d03c84373a855824519c38aff is 175, key is test_row_0/A:col10/1731091227971/Put/seqid=0 2024-11-08T18:40:29,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742004_1180 (size=31105) 2024-11-08T18:40:29,101 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/9ed9549d03c84373a855824519c38aff 2024-11-08T18:40:29,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0a814f5ba2024e7090823e7b8ef6d1e1 is 50, key is test_row_0/B:col10/1731091227971/Put/seqid=0 2024-11-08T18:40:29,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742005_1181 (size=12151) 2024-11-08T18:40:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:29,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:29,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091289198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:29,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:29,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091289309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:29,310 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:29,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T18:40:29,519 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0a814f5ba2024e7090823e7b8ef6d1e1 2024-11-08T18:40:29,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:29,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091289519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:29,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:29,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:29,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/32894210415b4e3090944141e1ad8a31 is 50, key is test_row_0/C:col10/1731091227971/Put/seqid=0 2024-11-08T18:40:29,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742006_1182 (size=12151) 2024-11-08T18:40:29,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091289829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:29,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:29,952 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/32894210415b4e3090944141e1ad8a31 2024-11-08T18:40:29,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/9ed9549d03c84373a855824519c38aff as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/9ed9549d03c84373a855824519c38aff 2024-11-08T18:40:29,961 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/9ed9549d03c84373a855824519c38aff, entries=150, sequenceid=212, filesize=30.4 K 2024-11-08T18:40:29,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0a814f5ba2024e7090823e7b8ef6d1e1 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0a814f5ba2024e7090823e7b8ef6d1e1 2024-11-08T18:40:29,966 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0a814f5ba2024e7090823e7b8ef6d1e1, entries=150, sequenceid=212, filesize=11.9 K 2024-11-08T18:40:29,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/32894210415b4e3090944141e1ad8a31 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/32894210415b4e3090944141e1ad8a31 2024-11-08T18:40:29,972 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/32894210415b4e3090944141e1ad8a31, entries=150, sequenceid=212, filesize=11.9 K 2024-11-08T18:40:29,973 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 922ms, sequenceid=212, compaction requested=true 2024-11-08T18:40:29,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:29,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:29,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-08T18:40:29,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-08T18:40:29,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-08T18:40:29,977 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6920 sec 2024-11-08T18:40:29,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.6970 sec 2024-11-08T18:40:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:30,341 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:30,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T18:40:30,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108eb8ffeea5ea046648f386d58e71a749c_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091229194/Put/seqid=0 2024-11-08T18:40:30,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742007_1183 (size=14794) 2024-11-08T18:40:30,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:30,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091290401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:30,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:30,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-08T18:40:30,429 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:30,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-08T18:40:30,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T18:40:30,431 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:30,432 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:30,432 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:30,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:30,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091290509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:30,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T18:40:30,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:30,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:30,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:30,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:30,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091290719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:30,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:30,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:30,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:30,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:30,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T18:40:30,773 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:30,777 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108eb8ffeea5ea046648f386d58e71a749c_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eb8ffeea5ea046648f386d58e71a749c_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:30,778 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/cc0e0cbe4d874626ac6081eca41f22b1, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:30,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/cc0e0cbe4d874626ac6081eca41f22b1 is 175, key is test_row_0/A:col10/1731091229194/Put/seqid=0 2024-11-08T18:40:30,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742008_1184 (size=39749) 2024-11-08T18:40:30,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:30,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:30,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091291039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:31,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T18:40:31,182 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/cc0e0cbe4d874626ac6081eca41f22b1 2024-11-08T18:40:31,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0c66501256df4efeb0267a0f1115b79e is 50, key is test_row_0/B:col10/1731091229194/Put/seqid=0 2024-11-08T18:40:31,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742009_1185 (size=12151) 2024-11-08T18:40:31,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,351 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:31,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:31,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:31,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:31,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091291549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:31,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T18:40:31,594 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0c66501256df4efeb0267a0f1115b79e 2024-11-08T18:40:31,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/80cfe38d39a94df7b33d2774e419fe2a is 50, key is test_row_0/C:col10/1731091229194/Put/seqid=0 2024-11-08T18:40:31,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742010_1186 (size=12151) 2024-11-08T18:40:31,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:31,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:31,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:31,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:31,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:31,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:31,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:32,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/80cfe38d39a94df7b33d2774e419fe2a 2024-11-08T18:40:32,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/cc0e0cbe4d874626ac6081eca41f22b1 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/cc0e0cbe4d874626ac6081eca41f22b1 2024-11-08T18:40:32,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/cc0e0cbe4d874626ac6081eca41f22b1, entries=200, sequenceid=233, filesize=38.8 K 2024-11-08T18:40:32,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0c66501256df4efeb0267a0f1115b79e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0c66501256df4efeb0267a0f1115b79e 2024-11-08T18:40:32,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0c66501256df4efeb0267a0f1115b79e, entries=150, sequenceid=233, filesize=11.9 K 2024-11-08T18:40:32,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/80cfe38d39a94df7b33d2774e419fe2a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/80cfe38d39a94df7b33d2774e419fe2a 2024-11-08T18:40:32,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/80cfe38d39a94df7b33d2774e419fe2a, entries=150, sequenceid=233, filesize=11.9 K 2024-11-08T18:40:32,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1685ms, sequenceid=233, compaction requested=true 2024-11-08T18:40:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:32,026 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:32,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:32,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:32,027 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:32,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:32,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:32,029 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142084 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:32,029 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:32,029 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/A is initiating minor compaction (all files) 2024-11-08T18:40:32,029 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/B is initiating minor compaction (all files) 2024-11-08T18:40:32,029 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/A in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:32,029 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/B in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:32,029 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3666808d895846dba46f8f3d332ed200, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f9316d24bea04427a61b335095be3088, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/9ed9549d03c84373a855824519c38aff, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/cc0e0cbe4d874626ac6081eca41f22b1] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=138.8 K 2024-11-08T18:40:32,029 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/15ff038fb9aa4d9da558c26620b08352, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/f200eac8d2cb474e8e399e8d6b7a1dc9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0a814f5ba2024e7090823e7b8ef6d1e1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0c66501256df4efeb0267a0f1115b79e] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=47.8 K 2024-11-08T18:40:32,029 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:32,029 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3666808d895846dba46f8f3d332ed200, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f9316d24bea04427a61b335095be3088, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/9ed9549d03c84373a855824519c38aff, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/cc0e0cbe4d874626ac6081eca41f22b1] 2024-11-08T18:40:32,029 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3666808d895846dba46f8f3d332ed200, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731091225956 2024-11-08T18:40:32,029 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 15ff038fb9aa4d9da558c26620b08352, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731091225956 2024-11-08T18:40:32,030 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9316d24bea04427a61b335095be3088, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091227238 2024-11-08T18:40:32,030 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f200eac8d2cb474e8e399e8d6b7a1dc9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091227238 2024-11-08T18:40:32,030 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ed9549d03c84373a855824519c38aff, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731091227971 2024-11-08T18:40:32,030 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a814f5ba2024e7090823e7b8ef6d1e1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731091227971 2024-11-08T18:40:32,030 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c66501256df4efeb0267a0f1115b79e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731091229194 2024-11-08T18:40:32,030 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc0e0cbe4d874626ac6081eca41f22b1, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731091229184 2024-11-08T18:40:32,038 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:32,040 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#B#compaction#156 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:32,041 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/98b10399a5034735abdcee89ff0d745b is 50, key is test_row_0/B:col10/1731091229194/Put/seqid=0 2024-11-08T18:40:32,042 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411082ddc4adebe194e71a989bb2c4d406d40_2db46d8a8e52bdcd84049ec79d0d81b7 store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:32,046 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411082ddc4adebe194e71a989bb2c4d406d40_2db46d8a8e52bdcd84049ec79d0d81b7, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:32,046 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082ddc4adebe194e71a989bb2c4d406d40_2db46d8a8e52bdcd84049ec79d0d81b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:32,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742011_1187 (size=12663) 2024-11-08T18:40:32,057 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/98b10399a5034735abdcee89ff0d745b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/98b10399a5034735abdcee89ff0d745b 2024-11-08T18:40:32,064 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/B of 2db46d8a8e52bdcd84049ec79d0d81b7 into 98b10399a5034735abdcee89ff0d745b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:32,064 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:32,064 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/B, priority=12, startTime=1731091232027; duration=0sec 2024-11-08T18:40:32,065 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:32,065 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:B 2024-11-08T18:40:32,065 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:32,071 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:32,071 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/C is initiating minor compaction (all files) 2024-11-08T18:40:32,071 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/C in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:32,071 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/78d650118e64451f858ee217bf673ebf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8a1d65dd70484d458a41e2d0c7221aef, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/32894210415b4e3090944141e1ad8a31, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/80cfe38d39a94df7b33d2774e419fe2a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=47.8 K 2024-11-08T18:40:32,071 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 78d650118e64451f858ee217bf673ebf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731091225956 2024-11-08T18:40:32,072 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a1d65dd70484d458a41e2d0c7221aef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731091227238 2024-11-08T18:40:32,072 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 32894210415b4e3090944141e1ad8a31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731091227971 2024-11-08T18:40:32,072 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 80cfe38d39a94df7b33d2774e419fe2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731091229194 2024-11-08T18:40:32,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742012_1188 (size=4469) 2024-11-08T18:40:32,079 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#A#compaction#157 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:32,079 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/e203522746eb4924bc94cfac0c394a76 is 175, key is test_row_0/A:col10/1731091229194/Put/seqid=0 2024-11-08T18:40:32,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742013_1189 (size=31617) 2024-11-08T18:40:32,083 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#C#compaction#158 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:32,084 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/5a6c156027e34fe1832383c9a2b21e2a is 50, key is test_row_0/C:col10/1731091229194/Put/seqid=0 2024-11-08T18:40:32,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742014_1190 (size=12663) 2024-11-08T18:40:32,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-08T18:40:32,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:32,116 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T18:40:32,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e872a846c50849fba54c53e123fa471f_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091230382/Put/seqid=0 2024-11-08T18:40:32,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742015_1191 (size=12304) 2024-11-08T18:40:32,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:32,148 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108e872a846c50849fba54c53e123fa471f_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e872a846c50849fba54c53e123fa471f_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:32,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0345ceab06f44d1b8b33cb1a5f4b4d5b, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0345ceab06f44d1b8b33cb1a5f4b4d5b is 175, key is test_row_0/A:col10/1731091230382/Put/seqid=0 2024-11-08T18:40:32,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742016_1192 (size=31105) 2024-11-08T18:40:32,157 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0345ceab06f44d1b8b33cb1a5f4b4d5b 2024-11-08T18:40:32,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/c761dd801e5e4240879eaa36c8c13e32 is 50, key is test_row_0/B:col10/1731091230382/Put/seqid=0 2024-11-08T18:40:32,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742017_1193 (size=12151) 2024-11-08T18:40:32,173 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/c761dd801e5e4240879eaa36c8c13e32 2024-11-08T18:40:32,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/ba192461deff48fabafc1832836ca36d is 50, key is test_row_0/C:col10/1731091230382/Put/seqid=0 2024-11-08T18:40:32,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742018_1194 (size=12151) 2024-11-08T18:40:32,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:32,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:32,489 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/e203522746eb4924bc94cfac0c394a76 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e203522746eb4924bc94cfac0c394a76 2024-11-08T18:40:32,498 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/5a6c156027e34fe1832383c9a2b21e2a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a6c156027e34fe1832383c9a2b21e2a 2024-11-08T18:40:32,500 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/A of 2db46d8a8e52bdcd84049ec79d0d81b7 into e203522746eb4924bc94cfac0c394a76(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:32,500 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:32,500 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/A, priority=12, startTime=1731091232026; duration=0sec 2024-11-08T18:40:32,500 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:32,500 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:A 2024-11-08T18:40:32,504 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/C of 2db46d8a8e52bdcd84049ec79d0d81b7 into 5a6c156027e34fe1832383c9a2b21e2a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:32,504 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:32,504 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/C, priority=12, startTime=1731091232027; duration=0sec 2024-11-08T18:40:32,504 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:32,504 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:C 2024-11-08T18:40:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091292543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,561 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091292543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091292562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091292564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,571 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091292570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,574 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:32,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,576 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T18:40:32,597 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/ba192461deff48fabafc1832836ca36d 2024-11-08T18:40:32,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0345ceab06f44d1b8b33cb1a5f4b4d5b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0345ceab06f44d1b8b33cb1a5f4b4d5b 2024-11-08T18:40:32,622 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0345ceab06f44d1b8b33cb1a5f4b4d5b, entries=150, sequenceid=249, filesize=30.4 K 2024-11-08T18:40:32,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/c761dd801e5e4240879eaa36c8c13e32 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c761dd801e5e4240879eaa36c8c13e32 2024-11-08T18:40:32,629 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c761dd801e5e4240879eaa36c8c13e32, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T18:40:32,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/ba192461deff48fabafc1832836ca36d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ba192461deff48fabafc1832836ca36d 2024-11-08T18:40:32,640 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ba192461deff48fabafc1832836ca36d, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T18:40:32,648 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 528ms, sequenceid=249, compaction requested=false 2024-11-08T18:40:32,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:32,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:32,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-08T18:40:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-08T18:40:32,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-08T18:40:32,655 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2180 sec 2024-11-08T18:40:32,662 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.2260 sec 2024-11-08T18:40:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:32,672 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:32,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:40:32,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091292695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,701 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091292698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bed11ca3520a4b21a2704264fa5fdb3a_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091232670/Put/seqid=0 2024-11-08T18:40:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091292703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091292701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,722 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742019_1195 (size=14994) 2024-11-08T18:40:32,763 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:32,768 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108bed11ca3520a4b21a2704264fa5fdb3a_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bed11ca3520a4b21a2704264fa5fdb3a_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:32,770 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/d2849a6447f447d4850fd811ddf72ae2, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:32,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/d2849a6447f447d4850fd811ddf72ae2 is 175, key is test_row_0/A:col10/1731091232670/Put/seqid=0 2024-11-08T18:40:32,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742020_1196 (size=39949) 2024-11-08T18:40:32,804 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/d2849a6447f447d4850fd811ddf72ae2 2024-11-08T18:40:32,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091292810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,812 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091292812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/dcbb69769b3f4160a3195e43ec8b1ebf is 50, key is test_row_0/B:col10/1731091232670/Put/seqid=0 2024-11-08T18:40:32,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091292830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091292830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:32,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:32,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742021_1197 (size=12301) 2024-11-08T18:40:32,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/dcbb69769b3f4160a3195e43ec8b1ebf 2024-11-08T18:40:32,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/a5e7e827297240efaed72125f6208b96 is 50, key is test_row_0/C:col10/1731091232670/Put/seqid=0 2024-11-08T18:40:32,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742022_1198 (size=12301) 2024-11-08T18:40:33,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091293020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091293020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091293039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091293039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,042 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,297 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/a5e7e827297240efaed72125f6208b96 2024-11-08T18:40:33,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/d2849a6447f447d4850fd811ddf72ae2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/d2849a6447f447d4850fd811ddf72ae2 2024-11-08T18:40:33,307 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/d2849a6447f447d4850fd811ddf72ae2, entries=200, sequenceid=274, filesize=39.0 K 2024-11-08T18:40:33,309 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/dcbb69769b3f4160a3195e43ec8b1ebf as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/dcbb69769b3f4160a3195e43ec8b1ebf 2024-11-08T18:40:33,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/dcbb69769b3f4160a3195e43ec8b1ebf, entries=150, sequenceid=274, filesize=12.0 K 2024-11-08T18:40:33,318 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/a5e7e827297240efaed72125f6208b96 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/a5e7e827297240efaed72125f6208b96 2024-11-08T18:40:33,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/a5e7e827297240efaed72125f6208b96, entries=150, sequenceid=274, filesize=12.0 K 2024-11-08T18:40:33,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 654ms, sequenceid=274, compaction requested=true 2024-11-08T18:40:33,326 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:33,327 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:33,327 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:33,327 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:33,328 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:33,328 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:33,328 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/A is initiating minor compaction (all files) 2024-11-08T18:40:33,328 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/B is initiating minor compaction (all files) 2024-11-08T18:40:33,328 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/B in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:33,328 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/98b10399a5034735abdcee89ff0d745b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c761dd801e5e4240879eaa36c8c13e32, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/dcbb69769b3f4160a3195e43ec8b1ebf] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=36.2 K 2024-11-08T18:40:33,328 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/A in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:33,328 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e203522746eb4924bc94cfac0c394a76, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0345ceab06f44d1b8b33cb1a5f4b4d5b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/d2849a6447f447d4850fd811ddf72ae2] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=100.3 K 2024-11-08T18:40:33,329 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:33,329 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e203522746eb4924bc94cfac0c394a76, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0345ceab06f44d1b8b33cb1a5f4b4d5b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/d2849a6447f447d4850fd811ddf72ae2] 2024-11-08T18:40:33,330 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 98b10399a5034735abdcee89ff0d745b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731091229194 2024-11-08T18:40:33,330 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e203522746eb4924bc94cfac0c394a76, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731091229194 2024-11-08T18:40:33,331 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting c761dd801e5e4240879eaa36c8c13e32, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091230382 2024-11-08T18:40:33,331 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0345ceab06f44d1b8b33cb1a5f4b4d5b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091230382 2024-11-08T18:40:33,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:33,332 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting dcbb69769b3f4160a3195e43ec8b1ebf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091232509 2024-11-08T18:40:33,332 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d2849a6447f447d4850fd811ddf72ae2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091232509 2024-11-08T18:40:33,332 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:33,332 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:40:33,353 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:33,363 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108c202d3724d414adeb18ce2db759fa3df_2db46d8a8e52bdcd84049ec79d0d81b7 store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:33,365 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108c202d3724d414adeb18ce2db759fa3df_2db46d8a8e52bdcd84049ec79d0d81b7, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:33,366 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c202d3724d414adeb18ce2db759fa3df_2db46d8a8e52bdcd84049ec79d0d81b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:33,369 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#B#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:33,370 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/229c941b9cc844ee891a199a28f060ac is 50, key is test_row_0/B:col10/1731091232670/Put/seqid=0 2024-11-08T18:40:33,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091293385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091293385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,394 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091293387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,394 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091293388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,398 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108564085fb18ff4e0fb84a9aa339b0b0fb_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091233332/Put/seqid=0 2024-11-08T18:40:33,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742023_1199 (size=4469) 2024-11-08T18:40:33,419 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#A#compaction#165 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:33,420 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0a6349ad41a94525ac815b53d458f7e3 is 175, key is test_row_0/A:col10/1731091232670/Put/seqid=0 2024-11-08T18:40:33,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742024_1200 (size=12915) 2024-11-08T18:40:33,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742025_1201 (size=12454) 2024-11-08T18:40:33,437 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:33,442 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108564085fb18ff4e0fb84a9aa339b0b0fb_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108564085fb18ff4e0fb84a9aa339b0b0fb_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:33,443 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/7691d559504f45f9bf68a97b6c0cbe83, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:33,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/7691d559504f45f9bf68a97b6c0cbe83 is 175, key is test_row_0/A:col10/1731091233332/Put/seqid=0 2024-11-08T18:40:33,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742026_1202 (size=31869) 2024-11-08T18:40:33,454 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0a6349ad41a94525ac815b53d458f7e3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0a6349ad41a94525ac815b53d458f7e3 2024-11-08T18:40:33,461 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/A of 2db46d8a8e52bdcd84049ec79d0d81b7 into 0a6349ad41a94525ac815b53d458f7e3(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:33,461 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:33,461 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/A, priority=13, startTime=1731091233326; duration=0sec 2024-11-08T18:40:33,461 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:33,461 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:A 2024-11-08T18:40:33,461 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:33,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742027_1203 (size=31255) 2024-11-08T18:40:33,463 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:33,463 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/C is initiating minor compaction (all files) 2024-11-08T18:40:33,463 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/C in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:33,463 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a6c156027e34fe1832383c9a2b21e2a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ba192461deff48fabafc1832836ca36d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/a5e7e827297240efaed72125f6208b96] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=36.2 K 2024-11-08T18:40:33,464 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a6c156027e34fe1832383c9a2b21e2a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731091229194 2024-11-08T18:40:33,464 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba192461deff48fabafc1832836ca36d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091230382 2024-11-08T18:40:33,464 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5e7e827297240efaed72125f6208b96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091232509 2024-11-08T18:40:33,479 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#C#compaction#168 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:33,479 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/e80784dc06fe4f8b99daaf5b6f78503f is 50, key is test_row_0/C:col10/1731091232670/Put/seqid=0 2024-11-08T18:40:33,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742028_1204 (size=12915) 2024-11-08T18:40:33,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091293500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,503 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091293500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091293500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091293500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,505 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091293710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091293710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091293710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:33,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091293710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:33,712 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:33,839 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/229c941b9cc844ee891a199a28f060ac as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/229c941b9cc844ee891a199a28f060ac 2024-11-08T18:40:33,846 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/B of 2db46d8a8e52bdcd84049ec79d0d81b7 into 229c941b9cc844ee891a199a28f060ac(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:33,846 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:33,846 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/B, priority=13, startTime=1731091233327; duration=0sec 2024-11-08T18:40:33,846 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:33,846 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:B 2024-11-08T18:40:33,864 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=289, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/7691d559504f45f9bf68a97b6c0cbe83 2024-11-08T18:40:33,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/b0eca8656eb14dcb8f840c51118e8ca2 is 50, key is test_row_0/B:col10/1731091233332/Put/seqid=0 2024-11-08T18:40:33,910 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/e80784dc06fe4f8b99daaf5b6f78503f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e80784dc06fe4f8b99daaf5b6f78503f 2024-11-08T18:40:33,915 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/C of 2db46d8a8e52bdcd84049ec79d0d81b7 into e80784dc06fe4f8b99daaf5b6f78503f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:33,915 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:33,915 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/C, priority=13, startTime=1731091233327; duration=0sec 2024-11-08T18:40:33,915 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:33,915 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:C 2024-11-08T18:40:33,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742029_1205 (size=12301) 2024-11-08T18:40:34,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091294020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091294020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091294020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091294022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/b0eca8656eb14dcb8f840c51118e8ca2 2024-11-08T18:40:34,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/e90d6099936140a1b07ab4bdc6171ea7 is 50, key is test_row_0/C:col10/1731091233332/Put/seqid=0 2024-11-08T18:40:34,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742030_1206 (size=12301) 2024-11-08T18:40:34,349 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=289 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/e90d6099936140a1b07ab4bdc6171ea7 2024-11-08T18:40:34,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/7691d559504f45f9bf68a97b6c0cbe83 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7691d559504f45f9bf68a97b6c0cbe83 2024-11-08T18:40:34,361 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7691d559504f45f9bf68a97b6c0cbe83, entries=150, sequenceid=289, filesize=30.5 K 2024-11-08T18:40:34,363 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/b0eca8656eb14dcb8f840c51118e8ca2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/b0eca8656eb14dcb8f840c51118e8ca2 2024-11-08T18:40:34,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/b0eca8656eb14dcb8f840c51118e8ca2, entries=150, sequenceid=289, filesize=12.0 K 2024-11-08T18:40:34,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/e90d6099936140a1b07ab4bdc6171ea7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e90d6099936140a1b07ab4bdc6171ea7 2024-11-08T18:40:34,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e90d6099936140a1b07ab4bdc6171ea7, entries=150, sequenceid=289, filesize=12.0 K 2024-11-08T18:40:34,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1048ms, sequenceid=289, compaction requested=false 2024-11-08T18:40:34,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:34,532 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:34,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T18:40:34,539 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081d823019b02b48edb43a02ea4bd26f7a_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091234530/Put/seqid=0 2024-11-08T18:40:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091294546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091294545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,547 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091294547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091294547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,551 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742031_1207 (size=12454) 2024-11-08T18:40:34,556 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:34,560 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081d823019b02b48edb43a02ea4bd26f7a_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081d823019b02b48edb43a02ea4bd26f7a_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:34,562 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/f0899d92e7294ba3aaea54f40e398d6e, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:34,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/f0899d92e7294ba3aaea54f40e398d6e is 175, key is test_row_0/A:col10/1731091234530/Put/seqid=0 2024-11-08T18:40:34,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091294579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742032_1208 (size=31255) 2024-11-08T18:40:34,581 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:34,581 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,582 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/f0899d92e7294ba3aaea54f40e398d6e 2024-11-08T18:40:34,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/82c551b19a254fd6a7e9838e3349c23b is 50, key is test_row_0/B:col10/1731091234530/Put/seqid=0 2024-11-08T18:40:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-08T18:40:34,600 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:34,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:34,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-08T18:40:34,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T18:40:34,603 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:34,604 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:34,604 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:34,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091294650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091294651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742033_1209 (size=12301) 2024-11-08T18:40:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091294659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091294659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T18:40:34,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T18:40:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:34,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:34,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:34,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:34,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091294860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091294860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091294869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:34,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091294879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:34,880 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:34,908 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T18:40:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:34,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:34,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:34,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:34,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T18:40:35,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T18:40:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:35,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/82c551b19a254fd6a7e9838e3349c23b 2024-11-08T18:40:35,091 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/eb37bdad08ca4158af114acb58c9e04c is 50, key is test_row_0/C:col10/1731091234530/Put/seqid=0 2024-11-08T18:40:35,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742034_1210 (size=12301) 2024-11-08T18:40:35,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091295170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091295170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091295179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091295189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T18:40:35,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T18:40:35,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:35,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T18:40:35,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:35,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:35,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/eb37bdad08ca4158af114acb58c9e04c 2024-11-08T18:40:35,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/f0899d92e7294ba3aaea54f40e398d6e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f0899d92e7294ba3aaea54f40e398d6e 2024-11-08T18:40:35,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f0899d92e7294ba3aaea54f40e398d6e, entries=150, sequenceid=316, filesize=30.5 K 2024-11-08T18:40:35,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/82c551b19a254fd6a7e9838e3349c23b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/82c551b19a254fd6a7e9838e3349c23b 2024-11-08T18:40:35,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/82c551b19a254fd6a7e9838e3349c23b, entries=150, sequenceid=316, filesize=12.0 K 2024-11-08T18:40:35,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/eb37bdad08ca4158af114acb58c9e04c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/eb37bdad08ca4158af114acb58c9e04c 2024-11-08T18:40:35,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/eb37bdad08ca4158af114acb58c9e04c, entries=150, sequenceid=316, filesize=12.0 K 2024-11-08T18:40:35,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1001ms, sequenceid=316, compaction requested=true 2024-11-08T18:40:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:35,534 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:35,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:35,534 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:35,536 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:35,536 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/A is initiating minor compaction (all files) 2024-11-08T18:40:35,536 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/A in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,536 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0a6349ad41a94525ac815b53d458f7e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7691d559504f45f9bf68a97b6c0cbe83, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f0899d92e7294ba3aaea54f40e398d6e] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=92.2 K 2024-11-08T18:40:35,536 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,536 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0a6349ad41a94525ac815b53d458f7e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7691d559504f45f9bf68a97b6c0cbe83, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f0899d92e7294ba3aaea54f40e398d6e] 2024-11-08T18:40:35,537 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:35,537 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/B is initiating minor compaction (all files) 2024-11-08T18:40:35,537 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/B in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,537 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/229c941b9cc844ee891a199a28f060ac, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/b0eca8656eb14dcb8f840c51118e8ca2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/82c551b19a254fd6a7e9838e3349c23b] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=36.6 K 2024-11-08T18:40:35,540 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a6349ad41a94525ac815b53d458f7e3, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091232509 2024-11-08T18:40:35,541 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 229c941b9cc844ee891a199a28f060ac, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091232509 2024-11-08T18:40:35,541 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7691d559504f45f9bf68a97b6c0cbe83, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731091232698 2024-11-08T18:40:35,542 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting b0eca8656eb14dcb8f840c51118e8ca2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731091232698 2024-11-08T18:40:35,542 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0899d92e7294ba3aaea54f40e398d6e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731091234530 2024-11-08T18:40:35,542 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 82c551b19a254fd6a7e9838e3349c23b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731091234530 2024-11-08T18:40:35,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-08T18:40:35,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,546 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:40:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:35,554 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:35,559 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:35,568 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#B#compaction#175 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:35,569 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/5eb62fe0acfb46c3a841bf0b40180b75 is 50, key is test_row_0/B:col10/1731091234530/Put/seqid=0 2024-11-08T18:40:35,575 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108eddfb47c5f7b4fa68cb0fd4e2882a104_2db46d8a8e52bdcd84049ec79d0d81b7 store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:35,577 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108eddfb47c5f7b4fa68cb0fd4e2882a104_2db46d8a8e52bdcd84049ec79d0d81b7, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:35,577 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108eddfb47c5f7b4fa68cb0fd4e2882a104_2db46d8a8e52bdcd84049ec79d0d81b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:35,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084795afc9411b4cefb4321ec78824aa73_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091234545/Put/seqid=0 2024-11-08T18:40:35,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742036_1212 (size=4469) 2024-11-08T18:40:35,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742035_1211 (size=13017) 2024-11-08T18:40:35,608 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#A#compaction#174 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:35,608 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/dd2c3165fabc4c54aa8806713b34afd2 is 175, key is test_row_0/A:col10/1731091234530/Put/seqid=0 2024-11-08T18:40:35,623 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/5eb62fe0acfb46c3a841bf0b40180b75 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/5eb62fe0acfb46c3a841bf0b40180b75 2024-11-08T18:40:35,635 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/B of 2db46d8a8e52bdcd84049ec79d0d81b7 into 5eb62fe0acfb46c3a841bf0b40180b75(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:35,636 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:35,636 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/B, priority=13, startTime=1731091235534; duration=0sec 2024-11-08T18:40:35,636 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:35,636 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:B 2024-11-08T18:40:35,636 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:35,640 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:35,640 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/C is initiating minor compaction (all files) 2024-11-08T18:40:35,640 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/C in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:35,641 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e80784dc06fe4f8b99daaf5b6f78503f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e90d6099936140a1b07ab4bdc6171ea7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/eb37bdad08ca4158af114acb58c9e04c] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=36.6 K 2024-11-08T18:40:35,641 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting e80784dc06fe4f8b99daaf5b6f78503f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731091232509 2024-11-08T18:40:35,642 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting e90d6099936140a1b07ab4bdc6171ea7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=289, earliestPutTs=1731091232698 2024-11-08T18:40:35,643 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting eb37bdad08ca4158af114acb58c9e04c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731091234530 2024-11-08T18:40:35,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742037_1213 (size=12454) 2024-11-08T18:40:35,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742038_1214 (size=31971) 2024-11-08T18:40:35,661 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:35,662 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/b242c02b936f463a9b9b05907c4951b0 is 50, key is test_row_0/C:col10/1731091234530/Put/seqid=0 2024-11-08T18:40:35,672 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/dd2c3165fabc4c54aa8806713b34afd2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dd2c3165fabc4c54aa8806713b34afd2 2024-11-08T18:40:35,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:35,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:35,686 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/A of 2db46d8a8e52bdcd84049ec79d0d81b7 into dd2c3165fabc4c54aa8806713b34afd2(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:35,686 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:35,686 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/A, priority=13, startTime=1731091235534; duration=0sec 2024-11-08T18:40:35,686 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:35,686 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:A 2024-11-08T18:40:35,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742039_1215 (size=13017) 2024-11-08T18:40:35,733 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/b242c02b936f463a9b9b05907c4951b0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b242c02b936f463a9b9b05907c4951b0 2024-11-08T18:40:35,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T18:40:35,768 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/C of 2db46d8a8e52bdcd84049ec79d0d81b7 into b242c02b936f463a9b9b05907c4951b0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:35,768 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:35,768 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/C, priority=13, startTime=1731091235554; duration=0sec 2024-11-08T18:40:35,768 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:35,768 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:C 2024-11-08T18:40:35,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091295795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091295797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,802 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,803 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,809 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091295803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091295805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091295910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091295910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,912 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091295919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091295919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:35,922 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:36,050 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411084795afc9411b4cefb4321ec78824aa73_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084795afc9411b4cefb4321ec78824aa73_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:36,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/34f5d6c494e64acea8c3a5372a43cdde, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:36,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/34f5d6c494e64acea8c3a5372a43cdde is 175, key is test_row_0/A:col10/1731091234545/Put/seqid=0 2024-11-08T18:40:36,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742040_1216 (size=31255) 2024-11-08T18:40:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091296120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091296120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,124 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091296129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,132 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091296129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091296430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091296430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,432 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091296439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091296442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/34f5d6c494e64acea8c3a5372a43cdde 2024-11-08T18:40:36,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/ae7f038e54ae42239abf09cdb8534fae is 50, key is test_row_0/B:col10/1731091234545/Put/seqid=0 2024-11-08T18:40:36,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742041_1217 (size=12301) 2024-11-08T18:40:36,568 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/ae7f038e54ae42239abf09cdb8534fae 2024-11-08T18:40:36,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/d4e9b0f1d6ec407492d6341f6760c23d is 50, key is test_row_0/C:col10/1731091234545/Put/seqid=0 2024-11-08T18:40:36,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742042_1218 (size=12301) 2024-11-08T18:40:36,589 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/d4e9b0f1d6ec407492d6341f6760c23d 2024-11-08T18:40:36,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/34f5d6c494e64acea8c3a5372a43cdde as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/34f5d6c494e64acea8c3a5372a43cdde 2024-11-08T18:40:36,605 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/34f5d6c494e64acea8c3a5372a43cdde, entries=150, sequenceid=329, filesize=30.5 K 2024-11-08T18:40:36,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/ae7f038e54ae42239abf09cdb8534fae as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/ae7f038e54ae42239abf09cdb8534fae 2024-11-08T18:40:36,616 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/ae7f038e54ae42239abf09cdb8534fae, entries=150, sequenceid=329, filesize=12.0 K 2024-11-08T18:40:36,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/d4e9b0f1d6ec407492d6341f6760c23d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d4e9b0f1d6ec407492d6341f6760c23d 2024-11-08T18:40:36,629 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d4e9b0f1d6ec407492d6341f6760c23d, entries=150, sequenceid=329, filesize=12.0 K 2024-11-08T18:40:36,633 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1087ms, sequenceid=329, compaction requested=false 2024-11-08T18:40:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:36,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-08T18:40:36,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-08T18:40:36,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-08T18:40:36,637 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0310 sec 2024-11-08T18:40:36,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0370 sec 2024-11-08T18:40:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-08T18:40:36,749 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:36,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:36,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-08T18:40:36,753 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:36,753 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:36,754 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:36,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T18:40:36,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T18:40:36,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-08T18:40:36,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:36,917 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:40:36,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c2474fe21711407f9f2b7f2b54c6f665_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091235793/Put/seqid=0 2024-11-08T18:40:36,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742043_1219 (size=12454) 2024-11-08T18:40:36,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:36,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:36,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091296965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091296966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,971 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091296969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,973 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:36,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:36,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091296971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:36,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T18:40:37,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091297079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091297080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091297080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091297080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,083 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,084 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091297289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091297289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,290 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091297290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091297290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,291 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:37,355 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c2474fe21711407f9f2b7f2b54c6f665_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c2474fe21711407f9f2b7f2b54c6f665_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:37,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:37,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9 is 175, key is test_row_0/A:col10/1731091235793/Put/seqid=0 2024-11-08T18:40:37,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742044_1220 (size=31255) 2024-11-08T18:40:37,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T18:40:37,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091297599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091297599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,601 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091297599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091297599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:37,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:37,761 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9 2024-11-08T18:40:37,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0d319c9a32694d95a2d4ffaee799d381 is 50, key is test_row_0/B:col10/1731091235793/Put/seqid=0 2024-11-08T18:40:37,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742045_1221 (size=12301) 2024-11-08T18:40:37,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T18:40:38,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091298110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091298110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:38,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091298110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,114 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091298110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,114 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:38,172 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0d319c9a32694d95a2d4ffaee799d381 2024-11-08T18:40:38,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/06264ecba5b446218f744f9998e9da9b is 50, key is test_row_0/C:col10/1731091235793/Put/seqid=0 2024-11-08T18:40:38,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742046_1222 (size=12301) 2024-11-08T18:40:38,583 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/06264ecba5b446218f744f9998e9da9b 2024-11-08T18:40:38,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9 2024-11-08T18:40:38,592 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9, entries=150, sequenceid=356, filesize=30.5 K 2024-11-08T18:40:38,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/0d319c9a32694d95a2d4ffaee799d381 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0d319c9a32694d95a2d4ffaee799d381 2024-11-08T18:40:38,597 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0d319c9a32694d95a2d4ffaee799d381, entries=150, sequenceid=356, filesize=12.0 K 2024-11-08T18:40:38,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/06264ecba5b446218f744f9998e9da9b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/06264ecba5b446218f744f9998e9da9b 2024-11-08T18:40:38,603 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/06264ecba5b446218f744f9998e9da9b, entries=150, sequenceid=356, filesize=12.0 K 2024-11-08T18:40:38,604 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1686ms, sequenceid=356, compaction requested=true 2024-11-08T18:40:38,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:38,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:38,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-08T18:40:38,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-08T18:40:38,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-08T18:40:38,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8510 sec 2024-11-08T18:40:38,608 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 1.8570 sec 2024-11-08T18:40:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:38,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:38,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:40:38,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108632b66d2a0cb4d8e8050b8a444487aae_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091236964/Put/seqid=0 2024-11-08T18:40:38,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742047_1223 (size=14994) 2024-11-08T18:40:38,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:38,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091298703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:38,820 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:38,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091298819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:38,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:38,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-08T18:40:38,899 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:38,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-08T18:40:38,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T18:40:38,901 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:38,902 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:38,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:39,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T18:40:39,024 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:39,028 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108632b66d2a0cb4d8e8050b8a444487aae_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108632b66d2a0cb4d8e8050b8a444487aae_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:39,029 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/faf6591994eb4c8ebb209a2ec7470367, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:39,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/faf6591994eb4c8ebb209a2ec7470367 is 175, key is test_row_0/A:col10/1731091236964/Put/seqid=0 2024-11-08T18:40:39,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:39,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091299029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,033 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:39,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742048_1224 (size=39949) 2024-11-08T18:40:39,040 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/faf6591994eb4c8ebb209a2ec7470367 2024-11-08T18:40:39,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/4f6a60022ecf476dbd42882cbda8a21c is 50, key is test_row_0/B:col10/1731091236964/Put/seqid=0 2024-11-08T18:40:39,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-08T18:40:39,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:39,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742049_1225 (size=12301) 2024-11-08T18:40:39,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54102 deadline: 1731091299119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54080 deadline: 1731091299119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:39,120 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:39,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54104 deadline: 1731091299119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:39,120 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:39,120 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:39,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:39,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54122 deadline: 1731091299129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:39,133 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2db46d8a8e52bdcd84049ec79d0d81b7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:39,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-08T18:40:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T18:40:39,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:39,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091299339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:39,360 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-08T18:40:39,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:39,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/4f6a60022ecf476dbd42882cbda8a21c 2024-11-08T18:40:39,464 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/5a75aef6218f4e4d995c43f1d0221ab0 is 50, key is test_row_0/C:col10/1731091236964/Put/seqid=0 2024-11-08T18:40:39,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742050_1226 (size=12301) 2024-11-08T18:40:39,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-08T18:40:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T18:40:39,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-08T18:40:39,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:39,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-08T18:40:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:39,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,819 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:39,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:39,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54162 deadline: 1731091299849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:39,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:40:39,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/5a75aef6218f4e4d995c43f1d0221ab0 2024-11-08T18:40:39,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/faf6591994eb4c8ebb209a2ec7470367 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/faf6591994eb4c8ebb209a2ec7470367 2024-11-08T18:40:39,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/faf6591994eb4c8ebb209a2ec7470367, entries=200, sequenceid=369, filesize=39.0 K 2024-11-08T18:40:39,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/4f6a60022ecf476dbd42882cbda8a21c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/4f6a60022ecf476dbd42882cbda8a21c 2024-11-08T18:40:39,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/4f6a60022ecf476dbd42882cbda8a21c, entries=150, sequenceid=369, filesize=12.0 K 2024-11-08T18:40:39,885 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/5a75aef6218f4e4d995c43f1d0221ab0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a75aef6218f4e4d995c43f1d0221ab0 2024-11-08T18:40:39,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a75aef6218f4e4d995c43f1d0221ab0, entries=150, sequenceid=369, filesize=12.0 K 2024-11-08T18:40:39,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1278ms, sequenceid=369, compaction requested=true 2024-11-08T18:40:39,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:39,890 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:39,890 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2db46d8a8e52bdcd84049ec79d0d81b7:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:39,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:39,891 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:39,891 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/B is initiating minor compaction (all files) 2024-11-08T18:40:39,891 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/B in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,891 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/5eb62fe0acfb46c3a841bf0b40180b75, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/ae7f038e54ae42239abf09cdb8534fae, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0d319c9a32694d95a2d4ffaee799d381, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/4f6a60022ecf476dbd42882cbda8a21c] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=48.8 K 2024-11-08T18:40:39,891 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134430 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:39,891 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/A is initiating minor compaction (all files) 2024-11-08T18:40:39,891 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eb62fe0acfb46c3a841bf0b40180b75, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731091234530 2024-11-08T18:40:39,891 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/A in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,892 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dd2c3165fabc4c54aa8806713b34afd2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/34f5d6c494e64acea8c3a5372a43cdde, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/faf6591994eb4c8ebb209a2ec7470367] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=131.3 K 2024-11-08T18:40:39,892 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,892 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dd2c3165fabc4c54aa8806713b34afd2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/34f5d6c494e64acea8c3a5372a43cdde, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/faf6591994eb4c8ebb209a2ec7470367] 2024-11-08T18:40:39,892 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting ae7f038e54ae42239abf09cdb8534fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731091234539 2024-11-08T18:40:39,892 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd2c3165fabc4c54aa8806713b34afd2, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731091234530 2024-11-08T18:40:39,892 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34f5d6c494e64acea8c3a5372a43cdde, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731091234539 2024-11-08T18:40:39,892 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d319c9a32694d95a2d4ffaee799d381, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731091235769 2024-11-08T18:40:39,892 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f6a60022ecf476dbd42882cbda8a21c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731091236964 2024-11-08T18:40:39,893 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 886d4bd0fd2b401cb97dc3b2e0b7cdd9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731091235769 2024-11-08T18:40:39,893 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting faf6591994eb4c8ebb209a2ec7470367, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731091236964 2024-11-08T18:40:39,899 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:39,900 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#B#compaction#186 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:39,901 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/7da651cb6e734120aa300ce725090ba8 is 50, key is test_row_0/B:col10/1731091236964/Put/seqid=0 2024-11-08T18:40:39,901 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110827679461250f475c84b002845b2a3637_2db46d8a8e52bdcd84049ec79d0d81b7 store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:39,903 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110827679461250f475c84b002845b2a3637_2db46d8a8e52bdcd84049ec79d0d81b7, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:39,903 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110827679461250f475c84b002845b2a3637_2db46d8a8e52bdcd84049ec79d0d81b7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:39,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742051_1227 (size=4469) 2024-11-08T18:40:39,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742052_1228 (size=13153) 2024-11-08T18:40:39,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-08T18:40:39,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:39,972 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:40:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088986391d7a8044ebb26587b96d8b3098_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091238696/Put/seqid=0 2024-11-08T18:40:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742053_1229 (size=12454) 2024-11-08T18:40:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T18:40:40,312 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#A#compaction#187 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:40,313 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/afba6f1c876c4cb2a62b466fe434ede8 is 175, key is test_row_0/A:col10/1731091236964/Put/seqid=0 2024-11-08T18:40:40,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742054_1230 (size=32107) 2024-11-08T18:40:40,323 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/7da651cb6e734120aa300ce725090ba8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7da651cb6e734120aa300ce725090ba8 2024-11-08T18:40:40,327 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/B of 2db46d8a8e52bdcd84049ec79d0d81b7 into 7da651cb6e734120aa300ce725090ba8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:40,327 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:40,328 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/B, priority=12, startTime=1731091239890; duration=0sec 2024-11-08T18:40:40,328 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:40,328 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:B 2024-11-08T18:40:40,328 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:40,329 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:40,329 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2db46d8a8e52bdcd84049ec79d0d81b7/C is initiating minor compaction (all files) 2024-11-08T18:40:40,329 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2db46d8a8e52bdcd84049ec79d0d81b7/C in TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:40,329 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b242c02b936f463a9b9b05907c4951b0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d4e9b0f1d6ec407492d6341f6760c23d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/06264ecba5b446218f744f9998e9da9b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a75aef6218f4e4d995c43f1d0221ab0] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp, totalSize=48.8 K 2024-11-08T18:40:40,329 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting b242c02b936f463a9b9b05907c4951b0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=316, earliestPutTs=1731091234530 2024-11-08T18:40:40,329 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d4e9b0f1d6ec407492d6341f6760c23d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731091234539 2024-11-08T18:40:40,330 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 06264ecba5b446218f744f9998e9da9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731091235769 2024-11-08T18:40:40,331 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a75aef6218f4e4d995c43f1d0221ab0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731091236964 2024-11-08T18:40:40,345 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2db46d8a8e52bdcd84049ec79d0d81b7#C#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:40,346 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/fd0e256eaa9e47dd94a9195c95fd4f45 is 50, key is test_row_0/C:col10/1731091236964/Put/seqid=0 2024-11-08T18:40:40,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742055_1231 (size=13153) 2024-11-08T18:40:40,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:40,385 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088986391d7a8044ebb26587b96d8b3098_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088986391d7a8044ebb26587b96d8b3098_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:40,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/e48d8fa8e76244ce840b0e4d9c37171b, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:40,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/e48d8fa8e76244ce840b0e4d9c37171b is 175, key is test_row_0/A:col10/1731091238696/Put/seqid=0 2024-11-08T18:40:40,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742056_1232 (size=31255) 2024-11-08T18:40:40,721 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/afba6f1c876c4cb2a62b466fe434ede8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/afba6f1c876c4cb2a62b466fe434ede8 2024-11-08T18:40:40,726 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/A of 2db46d8a8e52bdcd84049ec79d0d81b7 into afba6f1c876c4cb2a62b466fe434ede8(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:40,726 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:40,726 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/A, priority=12, startTime=1731091239890; duration=0sec 2024-11-08T18:40:40,726 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:40,726 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:A 2024-11-08T18:40:40,755 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/fd0e256eaa9e47dd94a9195c95fd4f45 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/fd0e256eaa9e47dd94a9195c95fd4f45 2024-11-08T18:40:40,759 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2db46d8a8e52bdcd84049ec79d0d81b7/C of 2db46d8a8e52bdcd84049ec79d0d81b7 into fd0e256eaa9e47dd94a9195c95fd4f45(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:40,759 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:40,759 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., storeName=2db46d8a8e52bdcd84049ec79d0d81b7/C, priority=12, startTime=1731091239890; duration=0sec 2024-11-08T18:40:40,759 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:40,760 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2db46d8a8e52bdcd84049ec79d0d81b7:C 2024-11-08T18:40:40,792 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=392, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/e48d8fa8e76244ce840b0e4d9c37171b 2024-11-08T18:40:40,793 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-08T18:40:40,793 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:40,793 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,793 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:40,794 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,795 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-08T18:40:40,795 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:40,795 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,795 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-08T18:40:40,795 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,795 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:40,795 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:40,795 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,795 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:40,799 INFO [Thread-654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-654. 2024-11-08T18:40:40,799 DEBUG [Thread-654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:40,799 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,799 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:40,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/d5326645836e49fdaed6d056c15fb7b7 is 50, key is test_row_0/B:col10/1731091238696/Put/seqid=0 2024-11-08T18:40:40,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742057_1233 (size=12301) 2024-11-08T18:40:40,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:40,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. as already flushing 2024-11-08T18:40:40,871 INFO [Thread-643 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-643. 2024-11-08T18:40:40,871 DEBUG [Thread-643 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:40,871 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,871 DEBUG [Thread-643 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:40,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T18:40:41,131 INFO [Thread-645 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-645. 2024-11-08T18:40:41,131 DEBUG [Thread-645 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:41,132 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:41,132 DEBUG [Thread-645 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:41,132 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:41,142 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-08T18:40:41,142 INFO [Thread-647 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-647. 2024-11-08T18:40:41,142 DEBUG [Thread-647 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:41,142 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:41,142 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:41,142 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:41,142 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:41,142 DEBUG [Thread-647 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:41,142 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:41,142 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:41,152 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-08T18:40:41,152 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:40:41,152 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:41,152 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:41,152 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:41,204 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/d5326645836e49fdaed6d056c15fb7b7 2024-11-08T18:40:41,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4abf7846b90c4211a6486d0647ef9c4c is 50, key is test_row_0/C:col10/1731091238696/Put/seqid=0 2024-11-08T18:40:41,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742058_1234 (size=12301) 2024-11-08T18:40:41,615 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4abf7846b90c4211a6486d0647ef9c4c 2024-11-08T18:40:41,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/e48d8fa8e76244ce840b0e4d9c37171b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e48d8fa8e76244ce840b0e4d9c37171b 2024-11-08T18:40:41,626 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e48d8fa8e76244ce840b0e4d9c37171b, entries=150, sequenceid=392, filesize=30.5 K 2024-11-08T18:40:41,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/d5326645836e49fdaed6d056c15fb7b7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/d5326645836e49fdaed6d056c15fb7b7 2024-11-08T18:40:41,632 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/d5326645836e49fdaed6d056c15fb7b7, entries=150, sequenceid=392, filesize=12.0 K 2024-11-08T18:40:41,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/4abf7846b90c4211a6486d0647ef9c4c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4abf7846b90c4211a6486d0647ef9c4c 2024-11-08T18:40:41,637 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4abf7846b90c4211a6486d0647ef9c4c, entries=150, sequenceid=392, filesize=12.0 K 2024-11-08T18:40:41,637 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T18:40:41,638 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=33.54 KB/34350 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1667ms, sequenceid=392, compaction requested=false 2024-11-08T18:40:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 2db46d8a8e52bdcd84049ec79d0d81b7: 2024-11-08T18:40:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:41,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-08T18:40:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-08T18:40:41,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-08T18:40:41,641 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7370 sec 2024-11-08T18:40:41,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 2.7410 sec 2024-11-08T18:40:43,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-08T18:40:43,059 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T18:40:43,059 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2935 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2959 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1284 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3852 rows 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1286 2024-11-08T18:40:43,060 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3855 rows 2024-11-08T18:40:43,060 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T18:40:43,060 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:40:43,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:43,060 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:43,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:40:43,061 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T18:40:43,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-08T18:40:43,064 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091243064"}]},"ts":"1731091243064"} 2024-11-08T18:40:43,066 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T18:40:43,066 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T18:40:43,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:40:43,068 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, UNASSIGN}] 2024-11-08T18:40:43,068 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, UNASSIGN 2024-11-08T18:40:43,069 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=2db46d8a8e52bdcd84049ec79d0d81b7, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:43,070 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, UNASSIGN because future has completed 2024-11-08T18:40:43,071 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:40:43,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:40:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-08T18:40:43,223 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(122): Close 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:43,223 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:40:43,223 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1722): Closing 2db46d8a8e52bdcd84049ec79d0d81b7, disabling compactions & flushes 2024-11-08T18:40:43,223 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:43,223 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:43,224 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. after waiting 0 ms 2024-11-08T18:40:43,224 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:43,224 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(2902): Flushing 2db46d8a8e52bdcd84049ec79d0d81b7 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T18:40:43,230 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a722a4a6c4144f2891999ee78082f8ed_2db46d8a8e52bdcd84049ec79d0d81b7 is 50, key is test_row_0/A:col10/1731091241150/Put/seqid=0 2024-11-08T18:40:43,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742059_1235 (size=12454) 2024-11-08T18:40:43,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-08T18:40:43,635 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:43,639 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a722a4a6c4144f2891999ee78082f8ed_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a722a4a6c4144f2891999ee78082f8ed_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:43,640 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0ba7ffa00c89446796602e61bdcb0a61, store: [table=TestAcidGuarantees family=A region=2db46d8a8e52bdcd84049ec79d0d81b7] 2024-11-08T18:40:43,641 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0ba7ffa00c89446796602e61bdcb0a61 is 175, key is test_row_0/A:col10/1731091241150/Put/seqid=0 2024-11-08T18:40:43,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742060_1236 (size=31255) 2024-11-08T18:40:43,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-08T18:40:44,047 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=403, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0ba7ffa00c89446796602e61bdcb0a61 2024-11-08T18:40:44,059 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/c992763aa7d745e9a41ce35a2dd84019 is 50, key is test_row_0/B:col10/1731091241150/Put/seqid=0 2024-11-08T18:40:44,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742061_1237 (size=12301) 2024-11-08T18:40:44,063 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/c992763aa7d745e9a41ce35a2dd84019 2024-11-08T18:40:44,070 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/545dd85e15934672adfb4ff04b00e2c5 is 50, key is test_row_0/C:col10/1731091241150/Put/seqid=0 2024-11-08T18:40:44,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742062_1238 (size=12301) 2024-11-08T18:40:44,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-08T18:40:44,475 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/545dd85e15934672adfb4ff04b00e2c5 2024-11-08T18:40:44,480 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/A/0ba7ffa00c89446796602e61bdcb0a61 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0ba7ffa00c89446796602e61bdcb0a61 2024-11-08T18:40:44,484 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0ba7ffa00c89446796602e61bdcb0a61, entries=150, sequenceid=403, filesize=30.5 K 2024-11-08T18:40:44,485 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/B/c992763aa7d745e9a41ce35a2dd84019 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c992763aa7d745e9a41ce35a2dd84019 2024-11-08T18:40:44,489 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c992763aa7d745e9a41ce35a2dd84019, entries=150, sequenceid=403, filesize=12.0 K 2024-11-08T18:40:44,490 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/.tmp/C/545dd85e15934672adfb4ff04b00e2c5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/545dd85e15934672adfb4ff04b00e2c5 2024-11-08T18:40:44,493 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/545dd85e15934672adfb4ff04b00e2c5, entries=150, sequenceid=403, filesize=12.0 K 2024-11-08T18:40:44,494 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1270ms, sequenceid=403, compaction requested=true 2024-11-08T18:40:44,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/2f020a7b8b974d3d87d7ea3e6e136f5c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/b3d8e6541ea445cbbe1ba941ec6e6bac, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/997ff78104b042c5928732337d8d35c2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/acb0c5bc81c44aa693250d655e094421, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/c93d1ec9ef844d05b6dd469a3708e2c3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3dd2f8d9745c48099f6abcd06ef53cc5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/154ecd7bc76440deae4679f786ef362f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/fb0f056e69ab41c4938b81233964f480, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7f97ae8ded484badb49b5d622cb6b49c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/aa108946d50e46ab9cd2db6510554919, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3666808d895846dba46f8f3d332ed200, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dca55cf0d9aa4c7a885aa91552fca18a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f9316d24bea04427a61b335095be3088, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/9ed9549d03c84373a855824519c38aff, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/cc0e0cbe4d874626ac6081eca41f22b1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e203522746eb4924bc94cfac0c394a76, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0345ceab06f44d1b8b33cb1a5f4b4d5b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/d2849a6447f447d4850fd811ddf72ae2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0a6349ad41a94525ac815b53d458f7e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7691d559504f45f9bf68a97b6c0cbe83, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dd2c3165fabc4c54aa8806713b34afd2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f0899d92e7294ba3aaea54f40e398d6e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/34f5d6c494e64acea8c3a5372a43cdde, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/faf6591994eb4c8ebb209a2ec7470367] to archive 2024-11-08T18:40:44,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:40:44,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/2f020a7b8b974d3d87d7ea3e6e136f5c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/2f020a7b8b974d3d87d7ea3e6e136f5c 2024-11-08T18:40:44,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/b3d8e6541ea445cbbe1ba941ec6e6bac to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/b3d8e6541ea445cbbe1ba941ec6e6bac 2024-11-08T18:40:44,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/997ff78104b042c5928732337d8d35c2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/997ff78104b042c5928732337d8d35c2 2024-11-08T18:40:44,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/acb0c5bc81c44aa693250d655e094421 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/acb0c5bc81c44aa693250d655e094421 2024-11-08T18:40:44,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/c93d1ec9ef844d05b6dd469a3708e2c3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/c93d1ec9ef844d05b6dd469a3708e2c3 2024-11-08T18:40:44,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3dd2f8d9745c48099f6abcd06ef53cc5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3dd2f8d9745c48099f6abcd06ef53cc5 2024-11-08T18:40:44,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/154ecd7bc76440deae4679f786ef362f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/154ecd7bc76440deae4679f786ef362f 2024-11-08T18:40:44,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/fb0f056e69ab41c4938b81233964f480 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/fb0f056e69ab41c4938b81233964f480 2024-11-08T18:40:44,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7f97ae8ded484badb49b5d622cb6b49c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7f97ae8ded484badb49b5d622cb6b49c 2024-11-08T18:40:44,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/aa108946d50e46ab9cd2db6510554919 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/aa108946d50e46ab9cd2db6510554919 2024-11-08T18:40:44,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3666808d895846dba46f8f3d332ed200 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/3666808d895846dba46f8f3d332ed200 2024-11-08T18:40:44,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dca55cf0d9aa4c7a885aa91552fca18a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dca55cf0d9aa4c7a885aa91552fca18a 2024-11-08T18:40:44,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f9316d24bea04427a61b335095be3088 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f9316d24bea04427a61b335095be3088 2024-11-08T18:40:44,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/9ed9549d03c84373a855824519c38aff to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/9ed9549d03c84373a855824519c38aff 2024-11-08T18:40:44,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/cc0e0cbe4d874626ac6081eca41f22b1 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/cc0e0cbe4d874626ac6081eca41f22b1 2024-11-08T18:40:44,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e203522746eb4924bc94cfac0c394a76 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e203522746eb4924bc94cfac0c394a76 2024-11-08T18:40:44,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0345ceab06f44d1b8b33cb1a5f4b4d5b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0345ceab06f44d1b8b33cb1a5f4b4d5b 2024-11-08T18:40:44,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/d2849a6447f447d4850fd811ddf72ae2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/d2849a6447f447d4850fd811ddf72ae2 2024-11-08T18:40:44,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0a6349ad41a94525ac815b53d458f7e3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0a6349ad41a94525ac815b53d458f7e3 2024-11-08T18:40:44,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7691d559504f45f9bf68a97b6c0cbe83 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/7691d559504f45f9bf68a97b6c0cbe83 2024-11-08T18:40:44,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dd2c3165fabc4c54aa8806713b34afd2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/dd2c3165fabc4c54aa8806713b34afd2 2024-11-08T18:40:44,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f0899d92e7294ba3aaea54f40e398d6e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/f0899d92e7294ba3aaea54f40e398d6e 2024-11-08T18:40:44,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/34f5d6c494e64acea8c3a5372a43cdde to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/34f5d6c494e64acea8c3a5372a43cdde 2024-11-08T18:40:44,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/886d4bd0fd2b401cb97dc3b2e0b7cdd9 2024-11-08T18:40:44,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/faf6591994eb4c8ebb209a2ec7470367 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/faf6591994eb4c8ebb209a2ec7470367 2024-11-08T18:40:44,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/92d496234918469d9399921782b88f54, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/794b021f764544d1bd2b8f10db2f81a1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7d11f12316464feab7dea8d65e8aa2c5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7f37d2dd439447a3adacd989b92b30b6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1c66dff60b5041b9bf40fa574fef95b4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/23e5d26d69fc408286b5d318e4661c7a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/3b59826514394186ad28369245de9934, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1dcd8ceedeac4eb0948125e0f88b1f83, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/40dfb84bbe6047669b8eb6d75d42f1ba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/bcbcf5cca0c7473887d38d2ee26dacc2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/15ff038fb9aa4d9da558c26620b08352, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/16146265cbc542e3abc29a4beb240949, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/f200eac8d2cb474e8e399e8d6b7a1dc9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0a814f5ba2024e7090823e7b8ef6d1e1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/98b10399a5034735abdcee89ff0d745b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0c66501256df4efeb0267a0f1115b79e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c761dd801e5e4240879eaa36c8c13e32, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/229c941b9cc844ee891a199a28f060ac, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/dcbb69769b3f4160a3195e43ec8b1ebf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/b0eca8656eb14dcb8f840c51118e8ca2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/5eb62fe0acfb46c3a841bf0b40180b75, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/82c551b19a254fd6a7e9838e3349c23b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/ae7f038e54ae42239abf09cdb8534fae, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0d319c9a32694d95a2d4ffaee799d381, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/4f6a60022ecf476dbd42882cbda8a21c] to archive 2024-11-08T18:40:44,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:40:44,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/92d496234918469d9399921782b88f54 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/92d496234918469d9399921782b88f54 2024-11-08T18:40:44,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/794b021f764544d1bd2b8f10db2f81a1 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/794b021f764544d1bd2b8f10db2f81a1 2024-11-08T18:40:44,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7d11f12316464feab7dea8d65e8aa2c5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7d11f12316464feab7dea8d65e8aa2c5 2024-11-08T18:40:44,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7f37d2dd439447a3adacd989b92b30b6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7f37d2dd439447a3adacd989b92b30b6 2024-11-08T18:40:44,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1c66dff60b5041b9bf40fa574fef95b4 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1c66dff60b5041b9bf40fa574fef95b4 2024-11-08T18:40:44,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/23e5d26d69fc408286b5d318e4661c7a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/23e5d26d69fc408286b5d318e4661c7a 2024-11-08T18:40:44,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/3b59826514394186ad28369245de9934 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/3b59826514394186ad28369245de9934 2024-11-08T18:40:44,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1dcd8ceedeac4eb0948125e0f88b1f83 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/1dcd8ceedeac4eb0948125e0f88b1f83 2024-11-08T18:40:44,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/40dfb84bbe6047669b8eb6d75d42f1ba to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/40dfb84bbe6047669b8eb6d75d42f1ba 2024-11-08T18:40:44,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/bcbcf5cca0c7473887d38d2ee26dacc2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/bcbcf5cca0c7473887d38d2ee26dacc2 2024-11-08T18:40:44,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/15ff038fb9aa4d9da558c26620b08352 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/15ff038fb9aa4d9da558c26620b08352 2024-11-08T18:40:44,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/16146265cbc542e3abc29a4beb240949 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/16146265cbc542e3abc29a4beb240949 2024-11-08T18:40:44,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/f200eac8d2cb474e8e399e8d6b7a1dc9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/f200eac8d2cb474e8e399e8d6b7a1dc9 2024-11-08T18:40:44,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0a814f5ba2024e7090823e7b8ef6d1e1 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0a814f5ba2024e7090823e7b8ef6d1e1 2024-11-08T18:40:44,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/98b10399a5034735abdcee89ff0d745b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/98b10399a5034735abdcee89ff0d745b 2024-11-08T18:40:44,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0c66501256df4efeb0267a0f1115b79e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0c66501256df4efeb0267a0f1115b79e 2024-11-08T18:40:44,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c761dd801e5e4240879eaa36c8c13e32 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c761dd801e5e4240879eaa36c8c13e32 2024-11-08T18:40:44,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/229c941b9cc844ee891a199a28f060ac to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/229c941b9cc844ee891a199a28f060ac 2024-11-08T18:40:44,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/dcbb69769b3f4160a3195e43ec8b1ebf to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/dcbb69769b3f4160a3195e43ec8b1ebf 2024-11-08T18:40:44,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/b0eca8656eb14dcb8f840c51118e8ca2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/b0eca8656eb14dcb8f840c51118e8ca2 2024-11-08T18:40:44,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/5eb62fe0acfb46c3a841bf0b40180b75 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/5eb62fe0acfb46c3a841bf0b40180b75 2024-11-08T18:40:44,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/82c551b19a254fd6a7e9838e3349c23b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/82c551b19a254fd6a7e9838e3349c23b 2024-11-08T18:40:44,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/ae7f038e54ae42239abf09cdb8534fae to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/ae7f038e54ae42239abf09cdb8534fae 2024-11-08T18:40:44,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0d319c9a32694d95a2d4ffaee799d381 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/0d319c9a32694d95a2d4ffaee799d381 2024-11-08T18:40:44,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/4f6a60022ecf476dbd42882cbda8a21c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/4f6a60022ecf476dbd42882cbda8a21c 2024-11-08T18:40:44,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/0356aed8c82d43a0b654ff5f8ed0bc1c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/91865a7796ae4612be7d59ac1019def5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8ba6d2a5370a4dd5bf183cd44c93fe46, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b2668ef4c99f4475b02896714d280c98, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4539076bf2a346f4aa7ae19f0f676a20, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/1722185e25774cb4b917d13eb7c703d2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ad531aca7b8340f79083d27c81e30808, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/702b7a4ce0834888a792f33cea19abf0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4c43f3d01eac4adb8c026ab265752a82, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/78d650118e64451f858ee217bf673ebf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/dfa00026b2c1438b9c92ef96b4495502, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8a1d65dd70484d458a41e2d0c7221aef, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/32894210415b4e3090944141e1ad8a31, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a6c156027e34fe1832383c9a2b21e2a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/80cfe38d39a94df7b33d2774e419fe2a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ba192461deff48fabafc1832836ca36d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e80784dc06fe4f8b99daaf5b6f78503f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/a5e7e827297240efaed72125f6208b96, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e90d6099936140a1b07ab4bdc6171ea7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b242c02b936f463a9b9b05907c4951b0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/eb37bdad08ca4158af114acb58c9e04c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d4e9b0f1d6ec407492d6341f6760c23d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/06264ecba5b446218f744f9998e9da9b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a75aef6218f4e4d995c43f1d0221ab0] to archive 2024-11-08T18:40:44,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:40:44,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/0356aed8c82d43a0b654ff5f8ed0bc1c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/0356aed8c82d43a0b654ff5f8ed0bc1c 2024-11-08T18:40:44,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/91865a7796ae4612be7d59ac1019def5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/91865a7796ae4612be7d59ac1019def5 2024-11-08T18:40:44,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8ba6d2a5370a4dd5bf183cd44c93fe46 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8ba6d2a5370a4dd5bf183cd44c93fe46 2024-11-08T18:40:44,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b2668ef4c99f4475b02896714d280c98 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b2668ef4c99f4475b02896714d280c98 2024-11-08T18:40:44,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4539076bf2a346f4aa7ae19f0f676a20 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4539076bf2a346f4aa7ae19f0f676a20 2024-11-08T18:40:44,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/1722185e25774cb4b917d13eb7c703d2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/1722185e25774cb4b917d13eb7c703d2 2024-11-08T18:40:44,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d891e2ff1dbb4ec1a1eee9f7d7c81c6b 2024-11-08T18:40:44,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ad531aca7b8340f79083d27c81e30808 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ad531aca7b8340f79083d27c81e30808 2024-11-08T18:40:44,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/702b7a4ce0834888a792f33cea19abf0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/702b7a4ce0834888a792f33cea19abf0 2024-11-08T18:40:44,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4c43f3d01eac4adb8c026ab265752a82 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4c43f3d01eac4adb8c026ab265752a82 2024-11-08T18:40:44,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/78d650118e64451f858ee217bf673ebf to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/78d650118e64451f858ee217bf673ebf 2024-11-08T18:40:44,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/dfa00026b2c1438b9c92ef96b4495502 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/dfa00026b2c1438b9c92ef96b4495502 2024-11-08T18:40:44,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8a1d65dd70484d458a41e2d0c7221aef to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/8a1d65dd70484d458a41e2d0c7221aef 2024-11-08T18:40:44,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/32894210415b4e3090944141e1ad8a31 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/32894210415b4e3090944141e1ad8a31 2024-11-08T18:40:44,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a6c156027e34fe1832383c9a2b21e2a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a6c156027e34fe1832383c9a2b21e2a 2024-11-08T18:40:44,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/80cfe38d39a94df7b33d2774e419fe2a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/80cfe38d39a94df7b33d2774e419fe2a 2024-11-08T18:40:44,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ba192461deff48fabafc1832836ca36d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/ba192461deff48fabafc1832836ca36d 2024-11-08T18:40:44,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e80784dc06fe4f8b99daaf5b6f78503f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e80784dc06fe4f8b99daaf5b6f78503f 2024-11-08T18:40:44,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/a5e7e827297240efaed72125f6208b96 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/a5e7e827297240efaed72125f6208b96 2024-11-08T18:40:44,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e90d6099936140a1b07ab4bdc6171ea7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/e90d6099936140a1b07ab4bdc6171ea7 2024-11-08T18:40:44,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b242c02b936f463a9b9b05907c4951b0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/b242c02b936f463a9b9b05907c4951b0 2024-11-08T18:40:44,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/eb37bdad08ca4158af114acb58c9e04c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/eb37bdad08ca4158af114acb58c9e04c 2024-11-08T18:40:44,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d4e9b0f1d6ec407492d6341f6760c23d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/d4e9b0f1d6ec407492d6341f6760c23d 2024-11-08T18:40:44,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/06264ecba5b446218f744f9998e9da9b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/06264ecba5b446218f744f9998e9da9b 2024-11-08T18:40:44,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a75aef6218f4e4d995c43f1d0221ab0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/5a75aef6218f4e4d995c43f1d0221ab0 2024-11-08T18:40:44,582 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/recovered.edits/406.seqid, newMaxSeqId=406, maxSeqId=4 2024-11-08T18:40:44,583 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. 2024-11-08T18:40:44,583 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] regionserver.HRegion(1676): Region close journal for 2db46d8a8e52bdcd84049ec79d0d81b7: Waiting for close lock at 1731091243223Running coprocessor pre-close hooks at 1731091243223Disabling compacts and flushes for region at 1731091243223Disabling writes for close at 1731091243224 (+1 ms)Obtaining lock to block concurrent updates at 1731091243224Preparing flush snapshotting stores in 2db46d8a8e52bdcd84049ec79d0d81b7 at 1731091243224Finished memstore snapshotting TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731091243224Flushing stores of TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7. at 1731091243225 (+1 ms)Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/A: creating writer at 1731091243225Flushing mob file 2db46d8a8e52bdcd84049ec79d0d81b7/A: appending metadata at 1731091243230 (+5 ms)Flushing mob file 2db46d8a8e52bdcd84049ec79d0d81b7/A: closing flushed file at 1731091243230Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/A: appending metadata at 1731091243640 (+410 ms)Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/A: closing flushed file at 1731091243640Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/B: creating writer at 1731091244056 (+416 ms)Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/B: appending metadata at 1731091244058 (+2 ms)Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/B: closing flushed file at 1731091244058Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/C: creating writer at 1731091244067 (+9 ms)Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/C: appending metadata at 1731091244070 (+3 ms)Flushing 2db46d8a8e52bdcd84049ec79d0d81b7/C: closing flushed file at 1731091244070Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23cb92d7: reopening flushed file at 1731091244479 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54718327: reopening flushed file at 1731091244484 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11b5ed98: reopening flushed file at 1731091244489 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 2db46d8a8e52bdcd84049ec79d0d81b7 in 1270ms, sequenceid=403, compaction requested=true at 1731091244494 (+5 ms)Writing region close event to WAL at 1731091244578 (+84 ms)Running coprocessor post-close hooks at 1731091244582 (+4 ms)Closed at 1731091244583 (+1 ms) 2024-11-08T18:40:44,585 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=59}] handler.UnassignRegionHandler(157): Closed 2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:44,586 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=2db46d8a8e52bdcd84049ec79d0d81b7, regionState=CLOSED 2024-11-08T18:40:44,587 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:40:44,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-08T18:40:44,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseRegionProcedure 2db46d8a8e52bdcd84049ec79d0d81b7, server=b75fe890c219,44447,1731091183487 in 1.5170 sec 2024-11-08T18:40:44,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-08T18:40:44,590 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2db46d8a8e52bdcd84049ec79d0d81b7, UNASSIGN in 1.5210 sec 2024-11-08T18:40:44,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-08T18:40:44,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5250 sec 2024-11-08T18:40:44,593 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091244593"}]},"ts":"1731091244593"} 2024-11-08T18:40:44,595 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T18:40:44,595 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T18:40:44,597 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5340 sec 2024-11-08T18:40:45,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-08T18:40:45,220 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:45,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T18:40:45,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:45,225 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=60, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-08T18:40:45,226 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=60, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:45,230 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,233 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/recovered.edits] 2024-11-08T18:40:45,235 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0ba7ffa00c89446796602e61bdcb0a61 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/0ba7ffa00c89446796602e61bdcb0a61 2024-11-08T18:40:45,237 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/afba6f1c876c4cb2a62b466fe434ede8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/afba6f1c876c4cb2a62b466fe434ede8 2024-11-08T18:40:45,238 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e48d8fa8e76244ce840b0e4d9c37171b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/A/e48d8fa8e76244ce840b0e4d9c37171b 2024-11-08T18:40:45,240 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7da651cb6e734120aa300ce725090ba8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/7da651cb6e734120aa300ce725090ba8 2024-11-08T18:40:45,242 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c992763aa7d745e9a41ce35a2dd84019 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/c992763aa7d745e9a41ce35a2dd84019 2024-11-08T18:40:45,243 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/d5326645836e49fdaed6d056c15fb7b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/B/d5326645836e49fdaed6d056c15fb7b7 2024-11-08T18:40:45,245 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4abf7846b90c4211a6486d0647ef9c4c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/4abf7846b90c4211a6486d0647ef9c4c 2024-11-08T18:40:45,246 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/545dd85e15934672adfb4ff04b00e2c5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/545dd85e15934672adfb4ff04b00e2c5 2024-11-08T18:40:45,247 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/fd0e256eaa9e47dd94a9195c95fd4f45 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/C/fd0e256eaa9e47dd94a9195c95fd4f45 2024-11-08T18:40:45,250 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/recovered.edits/406.seqid to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7/recovered.edits/406.seqid 2024-11-08T18:40:45,250 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,250 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T18:40:45,251 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T18:40:45,251 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T18:40:45,255 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110800b2f5417bc6419593d2b14ee079aa44_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110800b2f5417bc6419593d2b14ee079aa44_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,256 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108060cc887043b42bcba8c135ceb212bae_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108060cc887043b42bcba8c135ceb212bae_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,257 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a822233e8d54aefbe6b9bf085bf4335_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081a822233e8d54aefbe6b9bf085bf4335_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,258 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081d823019b02b48edb43a02ea4bd26f7a_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081d823019b02b48edb43a02ea4bd26f7a_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,259 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081e86e886c38240e8bf69467613695c3d_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081e86e886c38240e8bf69467613695c3d_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,260 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110821ec513486704763b3858bdcc444659d_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110821ec513486704763b3858bdcc444659d_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,261 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b97dc004c57439b879474b47b63d82d_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b97dc004c57439b879474b47b63d82d_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,262 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083f7efbd522624da2be75282a37c07617_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083f7efbd522624da2be75282a37c07617_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,264 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845d3173727c54275b4e53b2817bb8bda_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845d3173727c54275b4e53b2817bb8bda_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,265 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084795afc9411b4cefb4321ec78824aa73_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411084795afc9411b4cefb4321ec78824aa73_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,266 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108564085fb18ff4e0fb84a9aa339b0b0fb_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108564085fb18ff4e0fb84a9aa339b0b0fb_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,267 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108632b66d2a0cb4d8e8050b8a444487aae_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108632b66d2a0cb4d8e8050b8a444487aae_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,268 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088986391d7a8044ebb26587b96d8b3098_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088986391d7a8044ebb26587b96d8b3098_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,269 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a722a4a6c4144f2891999ee78082f8ed_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a722a4a6c4144f2891999ee78082f8ed_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,270 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108acb12ff7e1454406ac5820b0ac401bf6_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108acb12ff7e1454406ac5820b0ac401bf6_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,271 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bed11ca3520a4b21a2704264fa5fdb3a_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108bed11ca3520a4b21a2704264fa5fdb3a_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,272 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c2474fe21711407f9f2b7f2b54c6f665_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c2474fe21711407f9f2b7f2b54c6f665_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,273 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d881574c24af4352873da7de982b141b_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108d881574c24af4352873da7de982b141b_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,274 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e872a846c50849fba54c53e123fa471f_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108e872a846c50849fba54c53e123fa471f_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,275 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eb8ffeea5ea046648f386d58e71a749c_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108eb8ffeea5ea046648f386d58e71a749c_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,276 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fba102c1e83f4c7c99517445da4a327e_2db46d8a8e52bdcd84049ec79d0d81b7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108fba102c1e83f4c7c99517445da4a327e_2db46d8a8e52bdcd84049ec79d0d81b7 2024-11-08T18:40:45,277 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T18:40:45,279 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=60, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:45,281 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T18:40:45,283 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T18:40:45,285 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=60, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:45,285 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T18:40:45,285 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731091245285"}]},"ts":"9223372036854775807"} 2024-11-08T18:40:45,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T18:40:45,287 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2db46d8a8e52bdcd84049ec79d0d81b7, NAME => 'TestAcidGuarantees,,1731091218633.2db46d8a8e52bdcd84049ec79d0d81b7.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T18:40:45,287 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T18:40:45,287 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731091245287"}]},"ts":"9223372036854775807"} 2024-11-08T18:40:45,289 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T18:40:45,290 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=60, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:45,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-08T18:40:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-08T18:40:45,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T18:40:45,330 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:45,339 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=242 (was 237) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_671276091_22 at /127.0.0.1:55920 [Waiting for operation #143] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_733759443_22 at /127.0.0.1:55908 [Waiting for operation #159] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_733759443_22 at /127.0.0.1:59386 [Waiting for operation #228] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_671276091_22 at /127.0.0.1:59498 [Waiting for operation #203] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 459) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=524 (was 456) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3893 (was 4774) 2024-11-08T18:40:45,346 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=242, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=524, ProcessCount=11, AvailableMemoryMB=3892 2024-11-08T18:40:45,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:40:45,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:40:45,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T18:40:45,349 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T18:40:45,349 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:45,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 61 2024-11-08T18:40:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-08T18:40:45,350 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T18:40:45,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742063_1239 (size=959) 2024-11-08T18:40:45,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-08T18:40:45,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-08T18:40:45,757 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 2024-11-08T18:40:45,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742064_1240 (size=53) 2024-11-08T18:40:45,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:40:45,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0f4cfbb8cd1ae677542dbdc6afe60732, disabling compactions & flushes 2024-11-08T18:40:45,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:45,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:45,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. after waiting 0 ms 2024-11-08T18:40:45,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:45,764 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:45,764 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0f4cfbb8cd1ae677542dbdc6afe60732: Waiting for close lock at 1731091245764Disabling compacts and flushes for region at 1731091245764Disabling writes for close at 1731091245764Writing region close event to WAL at 1731091245764Closed at 1731091245764 2024-11-08T18:40:45,765 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T18:40:45,766 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731091245765"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731091245765"}]},"ts":"1731091245765"} 2024-11-08T18:40:45,767 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T18:40:45,768 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T18:40:45,768 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091245768"}]},"ts":"1731091245768"} 2024-11-08T18:40:45,770 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T18:40:45,771 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, ASSIGN}] 2024-11-08T18:40:45,772 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, ASSIGN 2024-11-08T18:40:45,773 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, ASSIGN; state=OFFLINE, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=false 2024-11-08T18:40:45,924 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=0f4cfbb8cd1ae677542dbdc6afe60732, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:45,925 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=62, ppid=61, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, ASSIGN because future has completed 2024-11-08T18:40:45,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:40:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-08T18:40:46,081 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:46,081 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7752): Opening region: {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:40:46,082 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,082 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:40:46,082 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7794): checking encryption for 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,082 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(7797): checking classloading for 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,084 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,085 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f4cfbb8cd1ae677542dbdc6afe60732 columnFamilyName A 2024-11-08T18:40:46,085 DEBUG [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:46,086 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] regionserver.HStore(327): Store=0f4cfbb8cd1ae677542dbdc6afe60732/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:46,086 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,089 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f4cfbb8cd1ae677542dbdc6afe60732 columnFamilyName B 2024-11-08T18:40:46,089 DEBUG [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:46,090 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] regionserver.HStore(327): Store=0f4cfbb8cd1ae677542dbdc6afe60732/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:46,090 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,091 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f4cfbb8cd1ae677542dbdc6afe60732 columnFamilyName C 2024-11-08T18:40:46,091 DEBUG [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:40:46,092 INFO [StoreOpener-0f4cfbb8cd1ae677542dbdc6afe60732-1 {}] regionserver.HStore(327): Store=0f4cfbb8cd1ae677542dbdc6afe60732/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:40:46,092 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1038): replaying wal for 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,092 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,093 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,094 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1048): stopping wal replay for 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,095 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1060): Cleaning up temporary data for 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,095 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:40:46,096 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1093): writing seq id for 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,099 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T18:40:46,099 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1114): Opened 0f4cfbb8cd1ae677542dbdc6afe60732; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73797846, jitterRate=0.09967359900474548}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:40:46,099 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,100 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegion(1006): Region open journal for 0f4cfbb8cd1ae677542dbdc6afe60732: Running coprocessor pre-open hook at 1731091246082Writing region info on filesystem at 1731091246082Initializing all the Stores at 1731091246083 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091246083Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091246083Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091246083Cleaning up temporary data from old regions at 1731091246095 (+12 ms)Running coprocessor post-open hooks at 1731091246099 (+4 ms)Region opened successfully at 1731091246100 (+1 ms) 2024-11-08T18:40:46,101 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., pid=63, masterSystemTime=1731091246078 2024-11-08T18:40:46,103 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:46,103 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=63}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:46,104 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=62 updating hbase:meta row=0f4cfbb8cd1ae677542dbdc6afe60732, regionState=OPEN, openSeqNum=2, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=63, ppid=62, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:40:46,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-08T18:40:46,108 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; OpenRegionProcedure 0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 in 180 msec 2024-11-08T18:40:46,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-08T18:40:46,110 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, ASSIGN in 338 msec 2024-11-08T18:40:46,111 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T18:40:46,112 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091246111"}]},"ts":"1731091246111"} 2024-11-08T18:40:46,113 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T18:40:46,114 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=61, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T18:40:46,116 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 767 msec 2024-11-08T18:40:46,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-11-08T18:40:46,491 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:46,493 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@781611af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,493 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,494 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,495 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59800d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,496 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,497 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,498 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@294d74df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,499 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,502 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,503 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T18:40:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is b75fe890c219,42293,1731091182659 2024-11-08T18:40:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6ec5b3a0 2024-11-08T18:40:46,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T18:40:46,505 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T18:40:46,507 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32f1f4e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,507 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,508 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,508 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@687d0292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,509 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,510 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,511 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7521cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,511 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cf99f0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,511 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,512 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149ddcd3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,514 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,515 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@179d1b4c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6594bb9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,517 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62386ff9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,518 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63804352, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,520 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61deb417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,520 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,520 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,521 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bd884c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,521 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,522 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,523 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d0eaf76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,524 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f17e9d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,524 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,524 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,525 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343ec856, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,527 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69ccb6dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,528 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7acbee0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,529 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,529 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,530 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34d4d173, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,530 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,531 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320e9f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,532 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46eaeeea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,532 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,532 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,532 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67654c96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,534 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18d29809, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bee866e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,535 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,535 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24353083, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,537 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@194c4a76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,538 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@148ac527, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,538 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,538 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,538 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dda5710, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,540 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,540 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f5ffa67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4199deaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,541 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:40:46,541 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:40:46,541 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ee1ab2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,541 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:40:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:40:46,542 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:40:46,542 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:40:46,543 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d377288, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:40:46,543 DEBUG [Thread-1124 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,543 DEBUG [Thread-1141 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,543 DEBUG [Thread-1130 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,543 DEBUG [Thread-1126 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,543 DEBUG [Thread-1139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,543 DEBUG [Thread-1137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,544 DEBUG [Thread-1122 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,544 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,544 DEBUG [Thread-1130 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,544 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,545 DEBUG [Thread-1128 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-08T18:40:46,545 DEBUG [Thread-1135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,546 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36308, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,546 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,546 DEBUG [Thread-1133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:40:46,546 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T18:40:46,546 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36300, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:40:46,547 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:40:46,547 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,547 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:46,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,548 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,548 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:40:46,549 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:46,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:46,549 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,550 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,552 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:40:46,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:46,559 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:46,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:40:46,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/2872a575d8824aeebb870dcebaa5721d is 50, key is test_row_0/A:col10/1731091246559/Put/seqid=0 2024-11-08T18:40:46,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742065_1241 (size=16681) 2024-11-08T18:40:46,598 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/2872a575d8824aeebb870dcebaa5721d 2024-11-08T18:40:46,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091306595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091306595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,603 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091306596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091306596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091306597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/6a91d243250d48778214cd473d9681e3 is 50, key is test_row_0/B:col10/1731091246559/Put/seqid=0 2024-11-08T18:40:46,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T18:40:46,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742066_1242 (size=12001) 2024-11-08T18:40:46,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T18:40:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:46,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:46,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:46,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:46,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091306709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091306709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,711 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091306709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091306709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091306709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T18:40:46,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:46,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:46,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:46,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:46,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:46,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T18:40:46,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091306919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,920 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091306919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091306920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091306920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091306920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T18:40:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:47,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:47,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:47,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/6a91d243250d48778214cd473d9681e3 2024-11-08T18:40:47,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/219c1c7023724d2fade01a4d5291b3b4 is 50, key is test_row_0/C:col10/1731091246559/Put/seqid=0 2024-11-08T18:40:47,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742067_1243 (size=12001) 2024-11-08T18:40:47,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/219c1c7023724d2fade01a4d5291b3b4 2024-11-08T18:40:47,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/2872a575d8824aeebb870dcebaa5721d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/2872a575d8824aeebb870dcebaa5721d 2024-11-08T18:40:47,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/2872a575d8824aeebb870dcebaa5721d, entries=250, sequenceid=14, filesize=16.3 K 2024-11-08T18:40:47,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/6a91d243250d48778214cd473d9681e3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/6a91d243250d48778214cd473d9681e3 2024-11-08T18:40:47,101 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/6a91d243250d48778214cd473d9681e3, entries=150, sequenceid=14, filesize=11.7 K 2024-11-08T18:40:47,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/219c1c7023724d2fade01a4d5291b3b4 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/219c1c7023724d2fade01a4d5291b3b4 2024-11-08T18:40:47,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/219c1c7023724d2fade01a4d5291b3b4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-08T18:40:47,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 548ms, sequenceid=14, compaction requested=false 2024-11-08T18:40:47,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:47,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-08T18:40:47,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:47,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:40:47,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/4f29c4d4f40643389afd51b16080bfd3 is 50, key is test_row_0/A:col10/1731091246595/Put/seqid=0 2024-11-08T18:40:47,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T18:40:47,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742068_1244 (size=12001) 2024-11-08T18:40:47,185 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/4f29c4d4f40643389afd51b16080bfd3 2024-11-08T18:40:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/35888a5f095846abb20741b916d77044 is 50, key is test_row_0/B:col10/1731091246595/Put/seqid=0 2024-11-08T18:40:47,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742069_1245 (size=12001) 2024-11-08T18:40:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:47,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:47,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091307238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091307238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091307240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091307240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091307241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091307349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091307350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091307350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091307350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091307350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,354 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091307560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091307560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,563 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091307560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091307560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091307560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,564 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,565 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,618 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/35888a5f095846abb20741b916d77044 2024-11-08T18:40:47,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/9fb845530f5249569e5cbe8f36954029 is 50, key is test_row_0/C:col10/1731091246595/Put/seqid=0 2024-11-08T18:40:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742070_1246 (size=12001) 2024-11-08T18:40:47,641 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/9fb845530f5249569e5cbe8f36954029 2024-11-08T18:40:47,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/4f29c4d4f40643389afd51b16080bfd3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/4f29c4d4f40643389afd51b16080bfd3 2024-11-08T18:40:47,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/4f29c4d4f40643389afd51b16080bfd3, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T18:40:47,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/35888a5f095846abb20741b916d77044 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/35888a5f095846abb20741b916d77044 2024-11-08T18:40:47,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/35888a5f095846abb20741b916d77044, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T18:40:47,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/9fb845530f5249569e5cbe8f36954029 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9fb845530f5249569e5cbe8f36954029 2024-11-08T18:40:47,658 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9fb845530f5249569e5cbe8f36954029, entries=150, sequenceid=38, filesize=11.7 K 2024-11-08T18:40:47,659 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 497ms, sequenceid=38, compaction requested=false 2024-11-08T18:40:47,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:47,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:47,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-08T18:40:47,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-08T18:40:47,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-08T18:40:47,662 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1120 sec 2024-11-08T18:40:47,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.1190 sec 2024-11-08T18:40:47,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-08T18:40:47,679 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:47,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:47,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-08T18:40:47,681 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:47,682 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:47,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T18:40:47,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:47,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T18:40:47,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-08T18:40:47,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:47,834 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:40:47,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/226002f8e9584399a87b48c8441a80f6 is 50, key is test_row_0/A:col10/1731091247238/Put/seqid=0 2024-11-08T18:40:47,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742071_1247 (size=12001) 2024-11-08T18:40:47,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:47,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:47,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091307890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,894 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091307892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091307893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091307893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,898 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:47,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091307894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:47,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:47,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T18:40:48,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091307999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091307999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091307999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091308010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091308010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,012 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,013 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091308209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091308209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091308209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091308219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091308219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,222 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,249 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/226002f8e9584399a87b48c8441a80f6 2024-11-08T18:40:48,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/da55f1897f7043b19862e7166c97d875 is 50, key is test_row_0/B:col10/1731091247238/Put/seqid=0 2024-11-08T18:40:48,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742072_1248 (size=12001) 2024-11-08T18:40:48,275 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/da55f1897f7043b19862e7166c97d875 2024-11-08T18:40:48,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/f5a1135446794516a0128c8080be3d06 is 50, key is test_row_0/C:col10/1731091247238/Put/seqid=0 2024-11-08T18:40:48,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T18:40:48,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742073_1249 (size=12001) 2024-11-08T18:40:48,315 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/f5a1135446794516a0128c8080be3d06 2024-11-08T18:40:48,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/226002f8e9584399a87b48c8441a80f6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/226002f8e9584399a87b48c8441a80f6 2024-11-08T18:40:48,332 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/226002f8e9584399a87b48c8441a80f6, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T18:40:48,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/da55f1897f7043b19862e7166c97d875 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/da55f1897f7043b19862e7166c97d875 2024-11-08T18:40:48,348 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/da55f1897f7043b19862e7166c97d875, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T18:40:48,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/f5a1135446794516a0128c8080be3d06 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/f5a1135446794516a0128c8080be3d06 2024-11-08T18:40:48,360 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/f5a1135446794516a0128c8080be3d06, entries=150, sequenceid=51, filesize=11.7 K 2024-11-08T18:40:48,361 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 527ms, sequenceid=51, compaction requested=true 2024-11-08T18:40:48,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:48,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:48,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-08T18:40:48,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-08T18:40:48,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-08T18:40:48,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 682 msec 2024-11-08T18:40:48,368 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 687 msec 2024-11-08T18:40:48,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:48,523 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:48,523 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T18:40:48,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/8f742a80b9f54d78b5e5ee39de41e0cb is 50, key is test_row_0/A:col10/1731091247891/Put/seqid=0 2024-11-08T18:40:48,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742074_1250 (size=16681) 2024-11-08T18:40:48,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091308530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091308531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091308534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,538 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091308535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091308535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,539 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091308639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091308639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091308649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091308649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091308649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-08T18:40:48,819 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:48,820 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:48,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-08T18:40:48,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T18:40:48,821 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:48,822 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:48,823 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:48,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091308849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091308849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,853 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091308860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091308860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091308860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:48,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T18:40:48,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/8f742a80b9f54d78b5e5ee39de41e0cb 2024-11-08T18:40:48,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5858cf5b1f8b404a990744a513194713 is 50, key is test_row_0/B:col10/1731091247891/Put/seqid=0 2024-11-08T18:40:48,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742075_1251 (size=12001) 2024-11-08T18:40:48,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T18:40:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:48,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:48,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:48,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T18:40:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T18:40:49,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091309159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,161 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091309160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,162 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091309169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091309169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091309169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T18:40:49,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:49,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5858cf5b1f8b404a990744a513194713 2024-11-08T18:40:49,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/9c2730865b4146afabf1b5c36431394c is 50, key is test_row_0/C:col10/1731091247891/Put/seqid=0 2024-11-08T18:40:49,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742076_1252 (size=12001) 2024-11-08T18:40:49,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T18:40:49,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T18:40:49,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T18:40:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091309669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091309669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091309679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091309679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091309680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:49,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:49,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T18:40:49,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:49,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:49,764 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/9c2730865b4146afabf1b5c36431394c 2024-11-08T18:40:49,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/8f742a80b9f54d78b5e5ee39de41e0cb as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f742a80b9f54d78b5e5ee39de41e0cb 2024-11-08T18:40:49,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f742a80b9f54d78b5e5ee39de41e0cb, entries=250, sequenceid=77, filesize=16.3 K 2024-11-08T18:40:49,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5858cf5b1f8b404a990744a513194713 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5858cf5b1f8b404a990744a513194713 2024-11-08T18:40:49,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5858cf5b1f8b404a990744a513194713, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T18:40:49,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/9c2730865b4146afabf1b5c36431394c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9c2730865b4146afabf1b5c36431394c 2024-11-08T18:40:49,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9c2730865b4146afabf1b5c36431394c, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T18:40:49,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1259ms, sequenceid=77, compaction requested=true 2024-11-08T18:40:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:49,782 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:49,783 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:49,783 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:49,783 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:49,783 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:49,784 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/A is initiating minor compaction (all files) 2024-11-08T18:40:49,784 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/B is initiating minor compaction (all files) 2024-11-08T18:40:49,784 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/A in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,784 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/B in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,784 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/2872a575d8824aeebb870dcebaa5721d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/4f29c4d4f40643389afd51b16080bfd3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/226002f8e9584399a87b48c8441a80f6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f742a80b9f54d78b5e5ee39de41e0cb] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=56.0 K 2024-11-08T18:40:49,784 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/6a91d243250d48778214cd473d9681e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/35888a5f095846abb20741b916d77044, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/da55f1897f7043b19862e7166c97d875, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5858cf5b1f8b404a990744a513194713] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=46.9 K 2024-11-08T18:40:49,784 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a91d243250d48778214cd473d9681e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731091246551 2024-11-08T18:40:49,784 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2872a575d8824aeebb870dcebaa5721d, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731091246551 2024-11-08T18:40:49,784 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 35888a5f095846abb20741b916d77044, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731091246593 2024-11-08T18:40:49,785 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting da55f1897f7043b19862e7166c97d875, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731091247237 2024-11-08T18:40:49,785 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f29c4d4f40643389afd51b16080bfd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731091246593 2024-11-08T18:40:49,785 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5858cf5b1f8b404a990744a513194713, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091247891 2024-11-08T18:40:49,785 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 226002f8e9584399a87b48c8441a80f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731091247237 2024-11-08T18:40:49,785 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f742a80b9f54d78b5e5ee39de41e0cb, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091247891 2024-11-08T18:40:49,794 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#B#compaction#207 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:49,794 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/45857c049dbd45b5abe037ed32a5206f is 50, key is test_row_0/B:col10/1731091247891/Put/seqid=0 2024-11-08T18:40:49,809 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:49,810 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/7f311f474dc1446aa16e2a09649424ba is 50, key is test_row_0/A:col10/1731091247891/Put/seqid=0 2024-11-08T18:40:49,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742077_1253 (size=12139) 2024-11-08T18:40:49,830 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/45857c049dbd45b5abe037ed32a5206f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/45857c049dbd45b5abe037ed32a5206f 2024-11-08T18:40:49,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742078_1254 (size=12139) 2024-11-08T18:40:49,836 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/B of 0f4cfbb8cd1ae677542dbdc6afe60732 into 45857c049dbd45b5abe037ed32a5206f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:49,836 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:49,837 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/B, priority=12, startTime=1731091249782; duration=0sec 2024-11-08T18:40:49,837 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:49,837 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:B 2024-11-08T18:40:49,837 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:49,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:49,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/C is initiating minor compaction (all files) 2024-11-08T18:40:49,839 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/C in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,839 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/219c1c7023724d2fade01a4d5291b3b4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9fb845530f5249569e5cbe8f36954029, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/f5a1135446794516a0128c8080be3d06, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9c2730865b4146afabf1b5c36431394c] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=46.9 K 2024-11-08T18:40:49,839 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/7f311f474dc1446aa16e2a09649424ba as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7f311f474dc1446aa16e2a09649424ba 2024-11-08T18:40:49,839 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 219c1c7023724d2fade01a4d5291b3b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731091246551 2024-11-08T18:40:49,839 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb845530f5249569e5cbe8f36954029, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731091246593 2024-11-08T18:40:49,841 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f5a1135446794516a0128c8080be3d06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731091247237 2024-11-08T18:40:49,841 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c2730865b4146afabf1b5c36431394c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091247891 2024-11-08T18:40:49,848 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/A of 0f4cfbb8cd1ae677542dbdc6afe60732 into 7f311f474dc1446aa16e2a09649424ba(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:49,848 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:49,848 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/A, priority=12, startTime=1731091249782; duration=0sec 2024-11-08T18:40:49,848 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:49,848 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:A 2024-11-08T18:40:49,857 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#C#compaction#209 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:49,857 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/8e975f89a50044a889e7f485788a1997 is 50, key is test_row_0/C:col10/1731091247891/Put/seqid=0 2024-11-08T18:40:49,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742079_1255 (size=12139) 2024-11-08T18:40:49,878 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/8e975f89a50044a889e7f485788a1997 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/8e975f89a50044a889e7f485788a1997 2024-11-08T18:40:49,887 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/C of 0f4cfbb8cd1ae677542dbdc6afe60732 into 8e975f89a50044a889e7f485788a1997(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:49,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:49,887 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/C, priority=12, startTime=1731091249782; duration=0sec 2024-11-08T18:40:49,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:49,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:C 2024-11-08T18:40:49,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-08T18:40:49,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:49,892 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T18:40:49,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/aef37567f59a40ad96ea44ce9c1aa991 is 50, key is test_row_0/A:col10/1731091248534/Put/seqid=0 2024-11-08T18:40:49,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742080_1256 (size=12001) 2024-11-08T18:40:49,912 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/aef37567f59a40ad96ea44ce9c1aa991 2024-11-08T18:40:49,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f0777622835c485887581ca37066e4b9 is 50, key is test_row_0/B:col10/1731091248534/Put/seqid=0 2024-11-08T18:40:49,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742081_1257 (size=12001) 2024-11-08T18:40:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T18:40:50,107 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T18:40:50,363 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f0777622835c485887581ca37066e4b9 2024-11-08T18:40:50,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/84fb094e46fb43ce83eae0844a7879d9 is 50, key is test_row_0/C:col10/1731091248534/Put/seqid=0 2024-11-08T18:40:50,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742082_1258 (size=12001) 2024-11-08T18:40:50,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:50,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:50,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091310706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091310706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091310707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091310708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091310711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,716 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,777 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/84fb094e46fb43ce83eae0844a7879d9 2024-11-08T18:40:50,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/aef37567f59a40ad96ea44ce9c1aa991 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/aef37567f59a40ad96ea44ce9c1aa991 2024-11-08T18:40:50,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/aef37567f59a40ad96ea44ce9c1aa991, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T18:40:50,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f0777622835c485887581ca37066e4b9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f0777622835c485887581ca37066e4b9 2024-11-08T18:40:50,790 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f0777622835c485887581ca37066e4b9, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T18:40:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/84fb094e46fb43ce83eae0844a7879d9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/84fb094e46fb43ce83eae0844a7879d9 2024-11-08T18:40:50,795 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/84fb094e46fb43ce83eae0844a7879d9, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T18:40:50,796 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 904ms, sequenceid=90, compaction requested=false 2024-11-08T18:40:50,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:50,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:50,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-08T18:40:50,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-08T18:40:50,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-08T18:40:50,799 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9750 sec 2024-11-08T18:40:50,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.9800 sec 2024-11-08T18:40:50,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:50,823 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:50,823 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T18:40:50,828 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/c1164a9bcdc84ae5a5bb5ecd1709b244 is 50, key is test_row_0/A:col10/1731091250822/Put/seqid=0 2024-11-08T18:40:50,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091310825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091310830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091310831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091310831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091310831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,836 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742083_1259 (size=14341) 2024-11-08T18:40:50,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/c1164a9bcdc84ae5a5bb5ecd1709b244 2024-11-08T18:40:50,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/28742d4edf274318b5551b3162f154fa is 50, key is test_row_0/B:col10/1731091250822/Put/seqid=0 2024-11-08T18:40:50,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742084_1260 (size=12001) 2024-11-08T18:40:50,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/28742d4edf274318b5551b3162f154fa 2024-11-08T18:40:50,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/7102010f84314198bedb85b1b0ef1298 is 50, key is test_row_0/C:col10/1731091250822/Put/seqid=0 2024-11-08T18:40:50,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742085_1261 (size=12001) 2024-11-08T18:40:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091310939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091310939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091310939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091310940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091310940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:50,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-08T18:40:50,969 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:50,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-08T18:40:50,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T18:40:50,971 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:50,972 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:50,972 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:51,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T18:40:51,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T18:40:51,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:51,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:51,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:51,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:51,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091311150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,151 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091311150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091311150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091311150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091311150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,153 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T18:40:51,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:51,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:51,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:51,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:51,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=60.38 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/7102010f84314198bedb85b1b0ef1298 2024-11-08T18:40:51,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/c1164a9bcdc84ae5a5bb5ecd1709b244 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/c1164a9bcdc84ae5a5bb5ecd1709b244 2024-11-08T18:40:51,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T18:40:51,291 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/c1164a9bcdc84ae5a5bb5ecd1709b244, entries=200, sequenceid=120, filesize=14.0 K 2024-11-08T18:40:51,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/28742d4edf274318b5551b3162f154fa as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/28742d4edf274318b5551b3162f154fa 2024-11-08T18:40:51,301 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/28742d4edf274318b5551b3162f154fa, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T18:40:51,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/7102010f84314198bedb85b1b0ef1298 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7102010f84314198bedb85b1b0ef1298 2024-11-08T18:40:51,306 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7102010f84314198bedb85b1b0ef1298, entries=150, sequenceid=120, filesize=11.7 K 2024-11-08T18:40:51,307 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~181.14 KB/185490, heapSize ~475.31 KB/486720, currentSize=33.54 KB/34350 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 484ms, sequenceid=120, compaction requested=true 2024-11-08T18:40:51,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:51,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:51,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:51,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:51,307 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:51,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:51,307 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:51,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:51,307 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:51,308 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38481 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:51,308 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:51,308 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/A is initiating minor compaction (all files) 2024-11-08T18:40:51,308 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/A in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:51,308 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7f311f474dc1446aa16e2a09649424ba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/aef37567f59a40ad96ea44ce9c1aa991, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/c1164a9bcdc84ae5a5bb5ecd1709b244] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=37.6 K 2024-11-08T18:40:51,309 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f311f474dc1446aa16e2a09649424ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091247891 2024-11-08T18:40:51,309 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/B is initiating minor compaction (all files) 2024-11-08T18:40:51,309 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/B in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:51,309 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting aef37567f59a40ad96ea44ce9c1aa991, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091248527 2024-11-08T18:40:51,309 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/45857c049dbd45b5abe037ed32a5206f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f0777622835c485887581ca37066e4b9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/28742d4edf274318b5551b3162f154fa] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=35.3 K 2024-11-08T18:40:51,310 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 45857c049dbd45b5abe037ed32a5206f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091247891 2024-11-08T18:40:51,310 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1164a9bcdc84ae5a5bb5ecd1709b244, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731091250821 2024-11-08T18:40:51,310 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f0777622835c485887581ca37066e4b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091248527 2024-11-08T18:40:51,310 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 28742d4edf274318b5551b3162f154fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731091250822 2024-11-08T18:40:51,318 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:51,318 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#B#compaction#217 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:51,318 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/16172339345e4450881e135f916f0ca3 is 50, key is test_row_0/A:col10/1731091250822/Put/seqid=0 2024-11-08T18:40:51,319 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/0aa6a7a76d96494aaf711c3cdf5562f2 is 50, key is test_row_0/B:col10/1731091250822/Put/seqid=0 2024-11-08T18:40:51,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742086_1262 (size=12241) 2024-11-08T18:40:51,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742087_1263 (size=12241) 2024-11-08T18:40:51,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-08T18:40:51,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:51,429 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T18:40:51,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/8f035b63a6704a45af876372a4365884 is 50, key is test_row_0/A:col10/1731091250829/Put/seqid=0 2024-11-08T18:40:51,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742088_1264 (size=9657) 2024-11-08T18:40:51,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:51,463 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:51,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091311482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091311483, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091311486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091311486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,490 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091311487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091311589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,590 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091311589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T18:40:51,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091311599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091311600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091311600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,740 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/16172339345e4450881e135f916f0ca3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/16172339345e4450881e135f916f0ca3 2024-11-08T18:40:51,741 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/0aa6a7a76d96494aaf711c3cdf5562f2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/0aa6a7a76d96494aaf711c3cdf5562f2 2024-11-08T18:40:51,744 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/B of 0f4cfbb8cd1ae677542dbdc6afe60732 into 0aa6a7a76d96494aaf711c3cdf5562f2(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:51,744 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/A of 0f4cfbb8cd1ae677542dbdc6afe60732 into 16172339345e4450881e135f916f0ca3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:51,745 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:51,745 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:51,745 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/A, priority=13, startTime=1731091251307; duration=0sec 2024-11-08T18:40:51,745 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/B, priority=13, startTime=1731091251307; duration=0sec 2024-11-08T18:40:51,745 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:51,745 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:A 2024-11-08T18:40:51,745 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:51,745 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:51,745 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:B 2024-11-08T18:40:51,746 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:51,746 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/C is initiating minor compaction (all files) 2024-11-08T18:40:51,746 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/C in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:51,746 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/8e975f89a50044a889e7f485788a1997, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/84fb094e46fb43ce83eae0844a7879d9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7102010f84314198bedb85b1b0ef1298] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=35.3 K 2024-11-08T18:40:51,746 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e975f89a50044a889e7f485788a1997, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091247891 2024-11-08T18:40:51,747 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84fb094e46fb43ce83eae0844a7879d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091248527 2024-11-08T18:40:51,747 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7102010f84314198bedb85b1b0ef1298, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731091250822 2024-11-08T18:40:51,753 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#C#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:51,753 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/aec091f62483435197d8e8c1edee0797 is 50, key is test_row_0/C:col10/1731091250822/Put/seqid=0 2024-11-08T18:40:51,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742089_1265 (size=12241) 2024-11-08T18:40:51,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091311799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091311799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,802 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091311810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091311810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091311810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:51,812 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:51,813 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:51,838 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/8f035b63a6704a45af876372a4365884 2024-11-08T18:40:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/636eaba890554018ab53d1d6b1f6a316 is 50, key is test_row_0/B:col10/1731091250829/Put/seqid=0 2024-11-08T18:40:51,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742090_1266 (size=9657) 2024-11-08T18:40:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T18:40:52,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091312109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091312109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091312120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,120 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091312120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091312120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,168 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/aec091f62483435197d8e8c1edee0797 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/aec091f62483435197d8e8c1edee0797 2024-11-08T18:40:52,172 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/C of 0f4cfbb8cd1ae677542dbdc6afe60732 into aec091f62483435197d8e8c1edee0797(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:52,172 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:52,172 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/C, priority=13, startTime=1731091251307; duration=0sec 2024-11-08T18:40:52,172 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:52,172 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:C 2024-11-08T18:40:52,248 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/636eaba890554018ab53d1d6b1f6a316 2024-11-08T18:40:52,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/54c701aeaadc4f21b4bd7b4a6dd1691f is 50, key is test_row_0/C:col10/1731091250829/Put/seqid=0 2024-11-08T18:40:52,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742091_1267 (size=9657) 2024-11-08T18:40:52,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091312619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091312619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091312629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091312629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091312629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:52,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:52,670 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/54c701aeaadc4f21b4bd7b4a6dd1691f 2024-11-08T18:40:52,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/8f035b63a6704a45af876372a4365884 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f035b63a6704a45af876372a4365884 2024-11-08T18:40:52,678 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f035b63a6704a45af876372a4365884, entries=100, sequenceid=128, filesize=9.4 K 2024-11-08T18:40:52,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/636eaba890554018ab53d1d6b1f6a316 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/636eaba890554018ab53d1d6b1f6a316 2024-11-08T18:40:52,683 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/636eaba890554018ab53d1d6b1f6a316, entries=100, sequenceid=128, filesize=9.4 K 2024-11-08T18:40:52,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/54c701aeaadc4f21b4bd7b4a6dd1691f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/54c701aeaadc4f21b4bd7b4a6dd1691f 2024-11-08T18:40:52,688 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/54c701aeaadc4f21b4bd7b4a6dd1691f, entries=100, sequenceid=128, filesize=9.4 K 2024-11-08T18:40:52,689 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=174.43 KB/178620 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1259ms, sequenceid=128, compaction requested=false 2024-11-08T18:40:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:52,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-08T18:40:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-08T18:40:52,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-08T18:40:52,692 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7180 sec 2024-11-08T18:40:52,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.7220 sec 2024-11-08T18:40:53,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-08T18:40:53,120 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:53,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:53,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-08T18:40:53,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T18:40:53,122 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:53,122 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:53,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:53,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T18:40:53,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-08T18:40:53,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:53,275 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-08T18:40:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b76ab2a215cc4517a37f637af09083cd is 50, key is test_row_0/A:col10/1731091251479/Put/seqid=0 2024-11-08T18:40:53,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742092_1268 (size=12151) 2024-11-08T18:40:53,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T18:40:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:53,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:53,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091313643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091313643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091313645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091313645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091313649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,652 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 0f4cfbb8cd1ae677542dbdc6afe60732 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,652 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,694 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b76ab2a215cc4517a37f637af09083cd 2024-11-08T18:40:53,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5eff674b78184d5c88d5b3bfe7652028 is 50, key is test_row_0/B:col10/1731091251479/Put/seqid=0 2024-11-08T18:40:53,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742093_1269 (size=12151) 2024-11-08T18:40:53,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T18:40:53,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091313759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091313759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091313759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091313759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091313969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091313969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091313969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:53,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091313969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,107 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5eff674b78184d5c88d5b3bfe7652028 2024-11-08T18:40:54,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/c578f654fdf84a68ba1654359d161d7a is 50, key is test_row_0/C:col10/1731091251479/Put/seqid=0 2024-11-08T18:40:54,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742094_1270 (size=12151) 2024-11-08T18:40:54,116 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/c578f654fdf84a68ba1654359d161d7a 2024-11-08T18:40:54,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b76ab2a215cc4517a37f637af09083cd as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b76ab2a215cc4517a37f637af09083cd 2024-11-08T18:40:54,123 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b76ab2a215cc4517a37f637af09083cd, entries=150, sequenceid=160, filesize=11.9 K 2024-11-08T18:40:54,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5eff674b78184d5c88d5b3bfe7652028 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5eff674b78184d5c88d5b3bfe7652028 2024-11-08T18:40:54,128 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5eff674b78184d5c88d5b3bfe7652028, entries=150, sequenceid=160, filesize=11.9 K 2024-11-08T18:40:54,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/c578f654fdf84a68ba1654359d161d7a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/c578f654fdf84a68ba1654359d161d7a 2024-11-08T18:40:54,132 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/c578f654fdf84a68ba1654359d161d7a, entries=150, sequenceid=160, filesize=11.9 K 2024-11-08T18:40:54,133 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 858ms, sequenceid=160, compaction requested=true 2024-11-08T18:40:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-08T18:40:54,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-08T18:40:54,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-08T18:40:54,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0120 sec 2024-11-08T18:40:54,138 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0170 sec 2024-11-08T18:40:54,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-08T18:40:54,260 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:54,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-08T18:40:54,261 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T18:40:54,262 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:54,262 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:54,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:54,285 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:54,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:40:54,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b802d0325cce4ec18d89e363ea8a7387 is 50, key is test_row_0/A:col10/1731091253642/Put/seqid=0 2024-11-08T18:40:54,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742095_1271 (size=14541) 2024-11-08T18:40:54,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091314314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,319 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091314318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091314319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091314319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T18:40:54,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091314430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091314430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,433 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091314430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091314430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,433 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:54,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:54,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,568 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T18:40:54,640 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091314639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,641 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091314639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091314639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091314639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,694 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b802d0325cce4ec18d89e363ea8a7387 2024-11-08T18:40:54,701 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/01711d79c77c44389e6f62a72e64b610 is 50, key is test_row_0/B:col10/1731091253642/Put/seqid=0 2024-11-08T18:40:54,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742096_1272 (size=12151) 2024-11-08T18:40:54,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:54,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:54,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T18:40:54,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091314949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091314949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091314950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091314950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:54,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:54,953 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:55,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/01711d79c77c44389e6f62a72e64b610 2024-11-08T18:40:55,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/2b2d0f3f9a97458699a97ae79bbde161 is 50, key is test_row_0/C:col10/1731091253642/Put/seqid=0 2024-11-08T18:40:55,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742097_1273 (size=12151) 2024-11-08T18:40:55,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,180 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T18:40:55,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091315459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:55,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091315459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:55,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091315460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:55,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091315461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:55,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:55,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:55,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/2b2d0f3f9a97458699a97ae79bbde161 2024-11-08T18:40:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b802d0325cce4ec18d89e363ea8a7387 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b802d0325cce4ec18d89e363ea8a7387 2024-11-08T18:40:55,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b802d0325cce4ec18d89e363ea8a7387, entries=200, sequenceid=171, filesize=14.2 K 2024-11-08T18:40:55,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/01711d79c77c44389e6f62a72e64b610 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01711d79c77c44389e6f62a72e64b610 2024-11-08T18:40:55,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01711d79c77c44389e6f62a72e64b610, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T18:40:55,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/2b2d0f3f9a97458699a97ae79bbde161 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/2b2d0f3f9a97458699a97ae79bbde161 2024-11-08T18:40:55,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/2b2d0f3f9a97458699a97ae79bbde161, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T18:40:55,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1282ms, sequenceid=171, compaction requested=true 2024-11-08T18:40:55,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:55,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:55,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:55,567 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:55,567 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:55,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:55,568 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:55,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:55,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:55,569 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:55,569 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/B is initiating minor compaction (all files) 2024-11-08T18:40:55,569 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/B in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,569 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/0aa6a7a76d96494aaf711c3cdf5562f2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/636eaba890554018ab53d1d6b1f6a316, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5eff674b78184d5c88d5b3bfe7652028, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01711d79c77c44389e6f62a72e64b610] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=45.1 K 2024-11-08T18:40:55,569 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48590 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:55,570 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/A is initiating minor compaction (all files) 2024-11-08T18:40:55,570 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/A in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,570 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/16172339345e4450881e135f916f0ca3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f035b63a6704a45af876372a4365884, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b76ab2a215cc4517a37f637af09083cd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b802d0325cce4ec18d89e363ea8a7387] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=47.5 K 2024-11-08T18:40:55,571 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16172339345e4450881e135f916f0ca3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731091250822 2024-11-08T18:40:55,571 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa6a7a76d96494aaf711c3cdf5562f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731091250822 2024-11-08T18:40:55,571 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 636eaba890554018ab53d1d6b1f6a316, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731091250829 2024-11-08T18:40:55,571 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f035b63a6704a45af876372a4365884, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731091250829 2024-11-08T18:40:55,572 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eff674b78184d5c88d5b3bfe7652028, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731091251479 2024-11-08T18:40:55,572 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b76ab2a215cc4517a37f637af09083cd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731091251479 2024-11-08T18:40:55,572 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 01711d79c77c44389e6f62a72e64b610, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091253642 2024-11-08T18:40:55,572 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b802d0325cce4ec18d89e363ea8a7387, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091253642 2024-11-08T18:40:55,587 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#A#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:55,588 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/ed3ef75018464809a32cdea6001e4291 is 50, key is test_row_0/A:col10/1731091253642/Put/seqid=0 2024-11-08T18:40:55,589 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#B#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:55,590 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/01ded053bef447a68fecee8dfaed815d is 50, key is test_row_0/B:col10/1731091253642/Put/seqid=0 2024-11-08T18:40:55,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742099_1275 (size=12527) 2024-11-08T18:40:55,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742098_1274 (size=12527) 2024-11-08T18:40:55,628 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/ed3ef75018464809a32cdea6001e4291 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/ed3ef75018464809a32cdea6001e4291 2024-11-08T18:40:55,634 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/A of 0f4cfbb8cd1ae677542dbdc6afe60732 into ed3ef75018464809a32cdea6001e4291(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:55,634 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:55,634 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/A, priority=12, startTime=1731091255567; duration=0sec 2024-11-08T18:40:55,634 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:55,634 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:A 2024-11-08T18:40:55,634 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:55,636 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46200 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:55,636 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/C is initiating minor compaction (all files) 2024-11-08T18:40:55,636 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/C in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,636 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/aec091f62483435197d8e8c1edee0797, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/54c701aeaadc4f21b4bd7b4a6dd1691f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/c578f654fdf84a68ba1654359d161d7a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/2b2d0f3f9a97458699a97ae79bbde161] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=45.1 K 2024-11-08T18:40:55,637 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting aec091f62483435197d8e8c1edee0797, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731091250822 2024-11-08T18:40:55,637 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c701aeaadc4f21b4bd7b4a6dd1691f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731091250829 2024-11-08T18:40:55,637 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c578f654fdf84a68ba1654359d161d7a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1731091251479 2024-11-08T18:40:55,638 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b2d0f3f9a97458699a97ae79bbde161, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091253642 2024-11-08T18:40:55,656 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#C#compaction#230 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:55,657 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/cadf38077f6243fa903f2d568d3f7aae is 50, key is test_row_0/C:col10/1731091253642/Put/seqid=0 2024-11-08T18:40:55,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:55,682 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:55,682 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T18:40:55,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:55,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742100_1276 (size=12527) 2024-11-08T18:40:55,694 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/a0954fa8fefb43ca852883bf56b17546 is 50, key is test_row_0/A:col10/1731091254318/Put/seqid=0 2024-11-08T18:40:55,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742101_1277 (size=12151) 2024-11-08T18:40:55,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091315727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:55,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091315839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:55,848 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:55,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:55,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:55,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:55,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:56,022 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/01ded053bef447a68fecee8dfaed815d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01ded053bef447a68fecee8dfaed815d 2024-11-08T18:40:56,036 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/B of 0f4cfbb8cd1ae677542dbdc6afe60732 into 01ded053bef447a68fecee8dfaed815d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:56,036 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:56,036 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/B, priority=12, startTime=1731091255567; duration=0sec 2024-11-08T18:40:56,036 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:56,036 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:B 2024-11-08T18:40:56,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091316059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,095 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/cadf38077f6243fa903f2d568d3f7aae as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cadf38077f6243fa903f2d568d3f7aae 2024-11-08T18:40:56,103 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/C of 0f4cfbb8cd1ae677542dbdc6afe60732 into cadf38077f6243fa903f2d568d3f7aae(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:56,103 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:56,103 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/C, priority=12, startTime=1731091255568; duration=0sec 2024-11-08T18:40:56,103 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:56,103 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:C 2024-11-08T18:40:56,113 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/a0954fa8fefb43ca852883bf56b17546 2024-11-08T18:40:56,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f7aebca2107e43e0b0a67dbdcea55faf is 50, key is test_row_0/B:col10/1731091254318/Put/seqid=0 2024-11-08T18:40:56,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:56,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:56,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:56,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:56,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742102_1278 (size=12151) 2024-11-08T18:40:56,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f7aebca2107e43e0b0a67dbdcea55faf 2024-11-08T18:40:56,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/09dca93440c7422f9b36ba7b1257c5fc is 50, key is test_row_0/C:col10/1731091254318/Put/seqid=0 2024-11-08T18:40:56,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742103_1279 (size=12151) 2024-11-08T18:40:56,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/09dca93440c7422f9b36ba7b1257c5fc 2024-11-08T18:40:56,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/a0954fa8fefb43ca852883bf56b17546 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/a0954fa8fefb43ca852883bf56b17546 2024-11-08T18:40:56,232 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/a0954fa8fefb43ca852883bf56b17546, entries=150, sequenceid=198, filesize=11.9 K 2024-11-08T18:40:56,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f7aebca2107e43e0b0a67dbdcea55faf as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f7aebca2107e43e0b0a67dbdcea55faf 2024-11-08T18:40:56,242 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f7aebca2107e43e0b0a67dbdcea55faf, entries=150, sequenceid=198, filesize=11.9 K 2024-11-08T18:40:56,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/09dca93440c7422f9b36ba7b1257c5fc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/09dca93440c7422f9b36ba7b1257c5fc 2024-11-08T18:40:56,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/09dca93440c7422f9b36ba7b1257c5fc, entries=150, sequenceid=198, filesize=11.9 K 2024-11-08T18:40:56,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 572ms, sequenceid=198, compaction requested=false 2024-11-08T18:40:56,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:56,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-08T18:40:56,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:56,298 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T18:40:56,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/7cebe61155714204ba1f29006ed7ae91 is 50, key is test_row_0/A:col10/1731091255723/Put/seqid=0 2024-11-08T18:40:56,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742104_1280 (size=12151) 2024-11-08T18:40:56,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/7cebe61155714204ba1f29006ed7ae91 2024-11-08T18:40:56,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/a6dceee991ff43cfaa528985f7ffd5c0 is 50, key is test_row_0/B:col10/1731091255723/Put/seqid=0 2024-11-08T18:40:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:56,372 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:56,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742105_1281 (size=12151) 2024-11-08T18:40:56,377 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/a6dceee991ff43cfaa528985f7ffd5c0 2024-11-08T18:40:56,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/16735ce68ab74b08acafd43ec67f7e8c is 50, key is test_row_0/C:col10/1731091255723/Put/seqid=0 2024-11-08T18:40:56,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T18:40:56,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742106_1282 (size=12151) 2024-11-08T18:40:56,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091316470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091316476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,484 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091316479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091316477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,485 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091316480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,486 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in 0f4cfbb8cd1ae677542dbdc6afe60732 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,487 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091316589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,591 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,591 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091316589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091316589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,592 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091316589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091316799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091316800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091316800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,803 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091316800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:56,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:56,817 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/16735ce68ab74b08acafd43ec67f7e8c 2024-11-08T18:40:56,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/7cebe61155714204ba1f29006ed7ae91 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7cebe61155714204ba1f29006ed7ae91 2024-11-08T18:40:56,827 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7cebe61155714204ba1f29006ed7ae91, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T18:40:56,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/a6dceee991ff43cfaa528985f7ffd5c0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/a6dceee991ff43cfaa528985f7ffd5c0 2024-11-08T18:40:56,833 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/a6dceee991ff43cfaa528985f7ffd5c0, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T18:40:56,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/16735ce68ab74b08acafd43ec67f7e8c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/16735ce68ab74b08acafd43ec67f7e8c 2024-11-08T18:40:56,838 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/16735ce68ab74b08acafd43ec67f7e8c, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T18:40:56,839 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 542ms, sequenceid=210, compaction requested=true 2024-11-08T18:40:56,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:56,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:56,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-08T18:40:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-08T18:40:56,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-08T18:40:56,845 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5800 sec 2024-11-08T18:40:56,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.5850 sec 2024-11-08T18:40:57,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:57,114 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:57,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T18:40:57,120 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b33d02daa84145e5b6f98006ce08096a is 50, key is test_row_0/A:col10/1731091256473/Put/seqid=0 2024-11-08T18:40:57,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091317119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091317122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,126 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091317125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091317125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742107_1283 (size=14541) 2024-11-08T18:40:57,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b33d02daa84145e5b6f98006ce08096a 2024-11-08T18:40:57,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/4d47b70f4cf14e129da92c296be26166 is 50, key is test_row_0/B:col10/1731091256473/Put/seqid=0 2024-11-08T18:40:57,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742108_1284 (size=12151) 2024-11-08T18:40:57,170 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/4d47b70f4cf14e129da92c296be26166 2024-11-08T18:40:57,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/be779b46a07a43769259436320522f80 is 50, key is test_row_0/C:col10/1731091256473/Put/seqid=0 2024-11-08T18:40:57,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742109_1285 (size=12151) 2024-11-08T18:40:57,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/be779b46a07a43769259436320522f80 2024-11-08T18:40:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b33d02daa84145e5b6f98006ce08096a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b33d02daa84145e5b6f98006ce08096a 2024-11-08T18:40:57,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b33d02daa84145e5b6f98006ce08096a, entries=200, sequenceid=239, filesize=14.2 K 2024-11-08T18:40:57,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/4d47b70f4cf14e129da92c296be26166 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/4d47b70f4cf14e129da92c296be26166 2024-11-08T18:40:57,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/4d47b70f4cf14e129da92c296be26166, entries=150, sequenceid=239, filesize=11.9 K 2024-11-08T18:40:57,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/be779b46a07a43769259436320522f80 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/be779b46a07a43769259436320522f80 2024-11-08T18:40:57,211 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/be779b46a07a43769259436320522f80, entries=150, sequenceid=239, filesize=11.9 K 2024-11-08T18:40:57,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=33.54 KB/34350 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 98ms, sequenceid=239, compaction requested=true 2024-11-08T18:40:57,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:57,213 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:57,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:57,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:57,213 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:57,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:57,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:57,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:57,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:57,215 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:57,215 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51370 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:57,215 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/B is initiating minor compaction (all files) 2024-11-08T18:40:57,215 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/A is initiating minor compaction (all files) 2024-11-08T18:40:57,215 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/A in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:57,216 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/ed3ef75018464809a32cdea6001e4291, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/a0954fa8fefb43ca852883bf56b17546, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7cebe61155714204ba1f29006ed7ae91, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b33d02daa84145e5b6f98006ce08096a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=50.2 K 2024-11-08T18:40:57,216 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed3ef75018464809a32cdea6001e4291, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091253642 2024-11-08T18:40:57,216 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0954fa8fefb43ca852883bf56b17546, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731091254318 2024-11-08T18:40:57,216 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cebe61155714204ba1f29006ed7ae91, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731091255702 2024-11-08T18:40:57,217 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b33d02daa84145e5b6f98006ce08096a, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731091256473 2024-11-08T18:40:57,222 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/B in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:57,223 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01ded053bef447a68fecee8dfaed815d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f7aebca2107e43e0b0a67dbdcea55faf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/a6dceee991ff43cfaa528985f7ffd5c0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/4d47b70f4cf14e129da92c296be26166] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=47.8 K 2024-11-08T18:40:57,223 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 01ded053bef447a68fecee8dfaed815d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091253642 2024-11-08T18:40:57,223 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f7aebca2107e43e0b0a67dbdcea55faf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731091254318 2024-11-08T18:40:57,224 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting a6dceee991ff43cfaa528985f7ffd5c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731091255702 2024-11-08T18:40:57,224 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d47b70f4cf14e129da92c296be26166, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731091256473 2024-11-08T18:40:57,226 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#A#compaction#240 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:57,226 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/90f89a2523e343fa890fdf755d17dbe1 is 50, key is test_row_0/A:col10/1731091256473/Put/seqid=0 2024-11-08T18:40:57,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742110_1286 (size=12663) 2024-11-08T18:40:57,235 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#B#compaction#241 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:57,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:57,236 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:57,236 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:40:57,239 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f5c1d581448a4628bf37aa9ca0c4d59d is 50, key is test_row_0/B:col10/1731091256473/Put/seqid=0 2024-11-08T18:40:57,248 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/90f89a2523e343fa890fdf755d17dbe1 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/90f89a2523e343fa890fdf755d17dbe1 2024-11-08T18:40:57,248 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/94ae3c6a5e364b839ad51efe9f3a873c is 50, key is test_row_0/A:col10/1731091257234/Put/seqid=0 2024-11-08T18:40:57,256 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/A of 0f4cfbb8cd1ae677542dbdc6afe60732 into 90f89a2523e343fa890fdf755d17dbe1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:57,256 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:57,256 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/A, priority=12, startTime=1731091257213; duration=0sec 2024-11-08T18:40:57,256 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:57,256 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:A 2024-11-08T18:40:57,257 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:40:57,258 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48980 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:40:57,258 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/C is initiating minor compaction (all files) 2024-11-08T18:40:57,258 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/C in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:57,258 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cadf38077f6243fa903f2d568d3f7aae, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/09dca93440c7422f9b36ba7b1257c5fc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/16735ce68ab74b08acafd43ec67f7e8c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/be779b46a07a43769259436320522f80] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=47.8 K 2024-11-08T18:40:57,259 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting cadf38077f6243fa903f2d568d3f7aae, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091253642 2024-11-08T18:40:57,259 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09dca93440c7422f9b36ba7b1257c5fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731091254318 2024-11-08T18:40:57,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742111_1287 (size=12663) 2024-11-08T18:40:57,260 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16735ce68ab74b08acafd43ec67f7e8c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731091255702 2024-11-08T18:40:57,260 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting be779b46a07a43769259436320522f80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731091256473 2024-11-08T18:40:57,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742112_1288 (size=12151) 2024-11-08T18:40:57,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/94ae3c6a5e364b839ad51efe9f3a873c 2024-11-08T18:40:57,270 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#C#compaction#243 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:57,270 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/70c1c3c639a94c5bb65cf1903e74238a is 50, key is test_row_0/C:col10/1731091256473/Put/seqid=0 2024-11-08T18:40:57,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/eb5269a210864d5782e249fb31222036 is 50, key is test_row_0/B:col10/1731091257234/Put/seqid=0 2024-11-08T18:40:57,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742113_1289 (size=12663) 2024-11-08T18:40:57,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742114_1290 (size=12151) 2024-11-08T18:40:57,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091317282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091317282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091317287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091317287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,292 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091317399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091317399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091317400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091317400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091317609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091317609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091317610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091317610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,613 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,665 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f5c1d581448a4628bf37aa9ca0c4d59d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f5c1d581448a4628bf37aa9ca0c4d59d 2024-11-08T18:40:57,669 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/B of 0f4cfbb8cd1ae677542dbdc6afe60732 into f5c1d581448a4628bf37aa9ca0c4d59d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:57,670 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:57,670 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/B, priority=12, startTime=1731091257213; duration=0sec 2024-11-08T18:40:57,670 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:57,670 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:B 2024-11-08T18:40:57,687 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/70c1c3c639a94c5bb65cf1903e74238a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/70c1c3c639a94c5bb65cf1903e74238a 2024-11-08T18:40:57,688 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/eb5269a210864d5782e249fb31222036 2024-11-08T18:40:57,693 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/C of 0f4cfbb8cd1ae677542dbdc6afe60732 into 70c1c3c639a94c5bb65cf1903e74238a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:57,693 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:57,693 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/C, priority=12, startTime=1731091257214; duration=0sec 2024-11-08T18:40:57,693 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:57,693 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:C 2024-11-08T18:40:57,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/cef7be2dfd7c4c7ea4eac3c741d3768f is 50, key is test_row_0/C:col10/1731091257234/Put/seqid=0 2024-11-08T18:40:57,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742115_1291 (size=12151) 2024-11-08T18:40:57,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091317919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091317919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091317920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091317920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:57,924 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/cef7be2dfd7c4c7ea4eac3c741d3768f 2024-11-08T18:40:58,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/94ae3c6a5e364b839ad51efe9f3a873c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/94ae3c6a5e364b839ad51efe9f3a873c 2024-11-08T18:40:58,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/94ae3c6a5e364b839ad51efe9f3a873c, entries=150, sequenceid=250, filesize=11.9 K 2024-11-08T18:40:58,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/eb5269a210864d5782e249fb31222036 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/eb5269a210864d5782e249fb31222036 2024-11-08T18:40:58,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/eb5269a210864d5782e249fb31222036, entries=150, sequenceid=250, filesize=11.9 K 2024-11-08T18:40:58,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/cef7be2dfd7c4c7ea4eac3c741d3768f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cef7be2dfd7c4c7ea4eac3c741d3768f 2024-11-08T18:40:58,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cef7be2dfd7c4c7ea4eac3c741d3768f, entries=150, sequenceid=250, filesize=11.9 K 2024-11-08T18:40:58,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 905ms, sequenceid=250, compaction requested=false 2024-11-08T18:40:58,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:58,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-08T18:40:58,430 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:40:58,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:40:58,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-08T18:40:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T18:40:58,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:40:58,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:40:58,432 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:40:58,432 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T18:40:58,432 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:40:58,433 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:40:58,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/15299a4ec9054f338ceaf1aa05de4d61 is 50, key is test_row_0/A:col10/1731091258431/Put/seqid=0 2024-11-08T18:40:58,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742116_1292 (size=12301) 2024-11-08T18:40:58,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091318438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,442 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091318439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091318440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,443 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091318442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091318499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,502 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in 0f4cfbb8cd1ae677542dbdc6afe60732 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:40:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T18:40:58,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091318549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091318549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091318549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,552 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091318549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:58,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:58,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:58,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:58,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:58,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:58,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T18:40:58,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091318759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091318760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091318760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091318760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:58,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/15299a4ec9054f338ceaf1aa05de4d61 2024-11-08T18:40:58,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f23f33a6e5e847e39326e3d530a5ea35 is 50, key is test_row_0/B:col10/1731091258431/Put/seqid=0 2024-11-08T18:40:58,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742117_1293 (size=12301) 2024-11-08T18:40:58,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:58,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:58,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:59,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:59,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T18:40:59,070 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091319069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:59,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091319069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:59,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091319069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:59,071 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091319070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:59,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:59,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:59,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,252 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f23f33a6e5e847e39326e3d530a5ea35 2024-11-08T18:40:59,259 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/ee63f62f22e94ec49b6bfbd216a224ea is 50, key is test_row_0/C:col10/1731091258431/Put/seqid=0 2024-11-08T18:40:59,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742118_1294 (size=12301) 2024-11-08T18:40:59,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:59,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:59,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:59,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:59,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,503 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T18:40:59,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091319579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:59,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091319579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:59,581 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091319580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:40:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091319580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:40:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:40:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:40:59,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:59,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:40:59,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:40:59,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=279 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/ee63f62f22e94ec49b6bfbd216a224ea 2024-11-08T18:40:59,666 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/15299a4ec9054f338ceaf1aa05de4d61 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/15299a4ec9054f338ceaf1aa05de4d61 2024-11-08T18:40:59,669 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/15299a4ec9054f338ceaf1aa05de4d61, entries=150, sequenceid=279, filesize=12.0 K 2024-11-08T18:40:59,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/f23f33a6e5e847e39326e3d530a5ea35 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f23f33a6e5e847e39326e3d530a5ea35 2024-11-08T18:40:59,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f23f33a6e5e847e39326e3d530a5ea35, entries=150, sequenceid=279, filesize=12.0 K 2024-11-08T18:40:59,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/ee63f62f22e94ec49b6bfbd216a224ea as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/ee63f62f22e94ec49b6bfbd216a224ea 2024-11-08T18:40:59,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/ee63f62f22e94ec49b6bfbd216a224ea, entries=150, sequenceid=279, filesize=12.0 K 2024-11-08T18:40:59,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1251ms, sequenceid=279, compaction requested=true 2024-11-08T18:40:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:40:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:59,683 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:40:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:40:59,683 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:40:59,683 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:59,685 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:59,685 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:59,685 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/A is initiating minor compaction (all files) 2024-11-08T18:40:59,685 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/B is initiating minor compaction (all files) 2024-11-08T18:40:59,685 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/B in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,685 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/A in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,686 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f5c1d581448a4628bf37aa9ca0c4d59d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/eb5269a210864d5782e249fb31222036, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f23f33a6e5e847e39326e3d530a5ea35] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.2 K 2024-11-08T18:40:59,686 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/90f89a2523e343fa890fdf755d17dbe1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/94ae3c6a5e364b839ad51efe9f3a873c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/15299a4ec9054f338ceaf1aa05de4d61] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.2 K 2024-11-08T18:40:59,686 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f5c1d581448a4628bf37aa9ca0c4d59d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731091256473 2024-11-08T18:40:59,686 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90f89a2523e343fa890fdf755d17dbe1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731091256473 2024-11-08T18:40:59,686 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting eb5269a210864d5782e249fb31222036, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731091257121 2024-11-08T18:40:59,686 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94ae3c6a5e364b839ad51efe9f3a873c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731091257121 2024-11-08T18:40:59,686 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f23f33a6e5e847e39326e3d530a5ea35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731091257279 2024-11-08T18:40:59,687 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15299a4ec9054f338ceaf1aa05de4d61, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731091257279 2024-11-08T18:40:59,693 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:59,694 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/675bd3bc2bef4207aadf67c67ee4fb46 is 50, key is test_row_0/B:col10/1731091258431/Put/seqid=0 2024-11-08T18:40:59,695 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#A#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:59,696 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/25900563c61b47119afb7ab5bf599aba is 50, key is test_row_0/A:col10/1731091258431/Put/seqid=0 2024-11-08T18:40:59,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742119_1295 (size=12915) 2024-11-08T18:40:59,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742120_1296 (size=12915) 2024-11-08T18:40:59,724 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/25900563c61b47119afb7ab5bf599aba as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/25900563c61b47119afb7ab5bf599aba 2024-11-08T18:40:59,728 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/A of 0f4cfbb8cd1ae677542dbdc6afe60732 into 25900563c61b47119afb7ab5bf599aba(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:40:59,728 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:40:59,728 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/A, priority=13, startTime=1731091259683; duration=0sec 2024-11-08T18:40:59,728 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:40:59,728 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:A 2024-11-08T18:40:59,728 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:40:59,729 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:40:59,729 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/C is initiating minor compaction (all files) 2024-11-08T18:40:59,730 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/C in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,730 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/70c1c3c639a94c5bb65cf1903e74238a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cef7be2dfd7c4c7ea4eac3c741d3768f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/ee63f62f22e94ec49b6bfbd216a224ea] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.2 K 2024-11-08T18:40:59,730 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c1c3c639a94c5bb65cf1903e74238a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731091256473 2024-11-08T18:40:59,730 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting cef7be2dfd7c4c7ea4eac3c741d3768f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731091257121 2024-11-08T18:40:59,730 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee63f62f22e94ec49b6bfbd216a224ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731091257279 2024-11-08T18:40:59,736 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:40:59,736 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/374f3b37544348bc80b9fff298b3b48e is 50, key is test_row_0/C:col10/1731091258431/Put/seqid=0 2024-11-08T18:40:59,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742121_1297 (size=12915) 2024-11-08T18:40:59,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-08T18:40:59,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:40:59,808 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-08T18:40:59,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/d1acb0eed3a04d41a9ae543f6904a4ee is 50, key is test_row_0/A:col10/1731091258437/Put/seqid=0 2024-11-08T18:40:59,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742122_1298 (size=12301) 2024-11-08T18:41:00,115 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/675bd3bc2bef4207aadf67c67ee4fb46 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/675bd3bc2bef4207aadf67c67ee4fb46 2024-11-08T18:41:00,119 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/B of 0f4cfbb8cd1ae677542dbdc6afe60732 into 675bd3bc2bef4207aadf67c67ee4fb46(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:00,119 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:00,119 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/B, priority=13, startTime=1731091259683; duration=0sec 2024-11-08T18:41:00,120 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:00,120 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:B 2024-11-08T18:41:00,163 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/374f3b37544348bc80b9fff298b3b48e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/374f3b37544348bc80b9fff298b3b48e 2024-11-08T18:41:00,168 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/C of 0f4cfbb8cd1ae677542dbdc6afe60732 into 374f3b37544348bc80b9fff298b3b48e(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:00,168 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:00,168 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/C, priority=13, startTime=1731091259683; duration=0sec 2024-11-08T18:41:00,168 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:00,168 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:C 2024-11-08T18:41:00,217 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/d1acb0eed3a04d41a9ae543f6904a4ee 2024-11-08T18:41:00,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/076235b2d3854793b07eb0d9635e4db0 is 50, key is test_row_0/B:col10/1731091258437/Put/seqid=0 2024-11-08T18:41:00,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742123_1299 (size=12301) 2024-11-08T18:41:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T18:41:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:00,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:41:00,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091320614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091320615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,619 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091320617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,621 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091320619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,632 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/076235b2d3854793b07eb0d9635e4db0 2024-11-08T18:41:00,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/1602f9fe4df144dd82bf6e25d04956cb is 50, key is test_row_0/C:col10/1731091258437/Put/seqid=0 2024-11-08T18:41:00,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742124_1300 (size=12301) 2024-11-08T18:41:00,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091320730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091320730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091320730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091320730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091320940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091320940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091320940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091320940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,042 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/1602f9fe4df144dd82bf6e25d04956cb 2024-11-08T18:41:01,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/d1acb0eed3a04d41a9ae543f6904a4ee as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/d1acb0eed3a04d41a9ae543f6904a4ee 2024-11-08T18:41:01,049 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/d1acb0eed3a04d41a9ae543f6904a4ee, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T18:41:01,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/076235b2d3854793b07eb0d9635e4db0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/076235b2d3854793b07eb0d9635e4db0 2024-11-08T18:41:01,054 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/076235b2d3854793b07eb0d9635e4db0, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T18:41:01,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/1602f9fe4df144dd82bf6e25d04956cb as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1602f9fe4df144dd82bf6e25d04956cb 2024-11-08T18:41:01,059 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1602f9fe4df144dd82bf6e25d04956cb, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T18:41:01,060 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1252ms, sequenceid=290, compaction requested=false 2024-11-08T18:41:01,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:01,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:01,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-08T18:41:01,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-08T18:41:01,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-08T18:41:01,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6280 sec 2024-11-08T18:41:01,066 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 2.6340 sec 2024-11-08T18:41:01,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:01,251 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:01,251 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-08T18:41:01,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091321257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,262 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091321259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091321260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091321261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/46879505afbc4921b195776ded81d01b is 50, key is test_row_0/A:col10/1731091261251/Put/seqid=0 2024-11-08T18:41:01,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742125_1301 (size=12301) 2024-11-08T18:41:01,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091321370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091321370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091321370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,374 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,374 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091321370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091321580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091321580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091321580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091321580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,583 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,676 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/46879505afbc4921b195776ded81d01b 2024-11-08T18:41:01,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/dcc217c1f2cd414eaf9a8a4b8647e781 is 50, key is test_row_0/B:col10/1731091261251/Put/seqid=0 2024-11-08T18:41:01,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742126_1302 (size=12301) 2024-11-08T18:41:01,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091321889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091321889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091321889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091321890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:01,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:02,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/dcc217c1f2cd414eaf9a8a4b8647e781 2024-11-08T18:41:02,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/009a203c24f7438b95e5cd3aec3e0db8 is 50, key is test_row_0/C:col10/1731091261251/Put/seqid=0 2024-11-08T18:41:02,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742127_1303 (size=12301) 2024-11-08T18:41:02,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091322399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,401 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:02,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091322399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:02,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:02,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091322399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091322399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:02,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/009a203c24f7438b95e5cd3aec3e0db8 2024-11-08T18:41:02,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:02,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36338 deadline: 1731091322509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:02,511 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in 0f4cfbb8cd1ae677542dbdc6afe60732 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:02,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:02,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/46879505afbc4921b195776ded81d01b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/46879505afbc4921b195776ded81d01b 2024-11-08T18:41:02,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/46879505afbc4921b195776ded81d01b, entries=150, sequenceid=320, filesize=12.0 K 2024-11-08T18:41:02,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/dcc217c1f2cd414eaf9a8a4b8647e781 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/dcc217c1f2cd414eaf9a8a4b8647e781 2024-11-08T18:41:02,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/dcc217c1f2cd414eaf9a8a4b8647e781, entries=150, sequenceid=320, filesize=12.0 K 2024-11-08T18:41:02,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/009a203c24f7438b95e5cd3aec3e0db8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/009a203c24f7438b95e5cd3aec3e0db8 2024-11-08T18:41:02,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/009a203c24f7438b95e5cd3aec3e0db8, entries=150, sequenceid=320, filesize=12.0 K 2024-11-08T18:41:02,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=40.25 KB/41220 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1279ms, sequenceid=320, compaction requested=true 2024-11-08T18:41:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:02,530 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:02,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:02,530 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:02,531 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:02,531 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:02,531 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/A is initiating minor compaction (all files) 2024-11-08T18:41:02,531 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/B is initiating minor compaction (all files) 2024-11-08T18:41:02,531 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/A in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:02,531 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/B in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:02,531 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/675bd3bc2bef4207aadf67c67ee4fb46, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/076235b2d3854793b07eb0d9635e4db0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/dcc217c1f2cd414eaf9a8a4b8647e781] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.6 K 2024-11-08T18:41:02,531 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/25900563c61b47119afb7ab5bf599aba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/d1acb0eed3a04d41a9ae543f6904a4ee, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/46879505afbc4921b195776ded81d01b] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.6 K 2024-11-08T18:41:02,533 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 675bd3bc2bef4207aadf67c67ee4fb46, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731091257279 2024-11-08T18:41:02,533 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25900563c61b47119afb7ab5bf599aba, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731091257279 2024-11-08T18:41:02,533 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1acb0eed3a04d41a9ae543f6904a4ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731091258432 2024-11-08T18:41:02,533 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 076235b2d3854793b07eb0d9635e4db0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731091258432 2024-11-08T18:41:02,534 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46879505afbc4921b195776ded81d01b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731091260617 2024-11-08T18:41:02,534 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting dcc217c1f2cd414eaf9a8a4b8647e781, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731091260617 2024-11-08T18:41:02,539 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#B#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:02,539 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#A#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:02,540 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/b27a15cc8aa14cbbb46cb85fb917562e is 50, key is test_row_0/B:col10/1731091261251/Put/seqid=0 2024-11-08T18:41:02,540 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b0439dfa33074a8ab132a9c5b760ab5b is 50, key is test_row_0/A:col10/1731091261251/Put/seqid=0 2024-11-08T18:41:02,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742128_1304 (size=13017) 2024-11-08T18:41:02,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742129_1305 (size=13017) 2024-11-08T18:41:02,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-08T18:41:02,599 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:02,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-08T18:41:02,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T18:41:02,602 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:02,602 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:02,602 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T18:41:02,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-08T18:41:02,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:02,755 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-08T18:41:02,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/41335f1e4467431880a7950c66bf7b51 is 50, key is test_row_0/A:col10/1731091261259/Put/seqid=0 2024-11-08T18:41:02,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742130_1306 (size=12301) 2024-11-08T18:41:02,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T18:41:02,951 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b0439dfa33074a8ab132a9c5b760ab5b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b0439dfa33074a8ab132a9c5b760ab5b 2024-11-08T18:41:02,952 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/b27a15cc8aa14cbbb46cb85fb917562e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/b27a15cc8aa14cbbb46cb85fb917562e 2024-11-08T18:41:02,957 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/B of 0f4cfbb8cd1ae677542dbdc6afe60732 into b27a15cc8aa14cbbb46cb85fb917562e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:02,957 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/A of 0f4cfbb8cd1ae677542dbdc6afe60732 into b0439dfa33074a8ab132a9c5b760ab5b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:02,957 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:02,957 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:02,957 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/B, priority=13, startTime=1731091262530; duration=0sec 2024-11-08T18:41:02,957 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/A, priority=13, startTime=1731091262530; duration=0sec 2024-11-08T18:41:02,957 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:02,957 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:B 2024-11-08T18:41:02,957 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:02,957 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:A 2024-11-08T18:41:02,957 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:02,958 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:02,958 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/C is initiating minor compaction (all files) 2024-11-08T18:41:02,958 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/C in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:02,958 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/374f3b37544348bc80b9fff298b3b48e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1602f9fe4df144dd82bf6e25d04956cb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/009a203c24f7438b95e5cd3aec3e0db8] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.6 K 2024-11-08T18:41:02,958 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 374f3b37544348bc80b9fff298b3b48e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=279, earliestPutTs=1731091257279 2024-11-08T18:41:02,958 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1602f9fe4df144dd82bf6e25d04956cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731091258432 2024-11-08T18:41:02,959 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 009a203c24f7438b95e5cd3aec3e0db8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731091260617 2024-11-08T18:41:02,964 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#C#compaction#261 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:02,965 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/e402e29cee524b18ab593b2e485e029e is 50, key is test_row_0/C:col10/1731091261251/Put/seqid=0 2024-11-08T18:41:02,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742131_1307 (size=13017) 2024-11-08T18:41:02,974 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/e402e29cee524b18ab593b2e485e029e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/e402e29cee524b18ab593b2e485e029e 2024-11-08T18:41:02,978 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/C of 0f4cfbb8cd1ae677542dbdc6afe60732 into e402e29cee524b18ab593b2e485e029e(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:02,978 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:02,978 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/C, priority=13, startTime=1731091262530; duration=0sec 2024-11-08T18:41:02,978 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:02,979 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:C 2024-11-08T18:41:03,162 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/41335f1e4467431880a7950c66bf7b51 2024-11-08T18:41:03,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5cf588beacc84038876c365dcba504fd is 50, key is test_row_0/B:col10/1731091261259/Put/seqid=0 2024-11-08T18:41:03,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742132_1308 (size=12301) 2024-11-08T18:41:03,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T18:41:03,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:41:03,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:03,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091323440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091323441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,446 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091323443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091323446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091323550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091323550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,551 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091323550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091323559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,561 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,573 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5cf588beacc84038876c365dcba504fd 2024-11-08T18:41:03,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/87768a98f2f94c8abad20bfef83cb290 is 50, key is test_row_0/C:col10/1731091261259/Put/seqid=0 2024-11-08T18:41:03,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742133_1309 (size=12301) 2024-11-08T18:41:03,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T18:41:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091323760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091323760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,762 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091323760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:03,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091323769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:03,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:03,983 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/87768a98f2f94c8abad20bfef83cb290 2024-11-08T18:41:03,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/41335f1e4467431880a7950c66bf7b51 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/41335f1e4467431880a7950c66bf7b51 2024-11-08T18:41:03,991 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/41335f1e4467431880a7950c66bf7b51, entries=150, sequenceid=329, filesize=12.0 K 2024-11-08T18:41:03,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/5cf588beacc84038876c365dcba504fd as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5cf588beacc84038876c365dcba504fd 2024-11-08T18:41:03,995 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5cf588beacc84038876c365dcba504fd, entries=150, sequenceid=329, filesize=12.0 K 2024-11-08T18:41:03,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/87768a98f2f94c8abad20bfef83cb290 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/87768a98f2f94c8abad20bfef83cb290 2024-11-08T18:41:04,000 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/87768a98f2f94c8abad20bfef83cb290, entries=150, sequenceid=329, filesize=12.0 K 2024-11-08T18:41:04,001 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=161.02 KB/164880 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1246ms, sequenceid=329, compaction requested=false 2024-11-08T18:41:04,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:04,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:04,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-08T18:41:04,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-08T18:41:04,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-08T18:41:04,004 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-08T18:41:04,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.4050 sec 2024-11-08T18:41:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:04,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:04,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-08T18:41:04,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/f3db5fbc2cd84942b8d1887b610b114a is 50, key is test_row_0/A:col10/1731091263438/Put/seqid=0 2024-11-08T18:41:04,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742134_1310 (size=12301) 2024-11-08T18:41:04,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091324077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091324077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,081 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091324078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091324079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,082 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091324190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091324190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091324190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091324399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091324400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091324400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/f3db5fbc2cd84942b8d1887b610b114a 2024-11-08T18:41:04,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/960ad7d324a343b08f27ef6c871f726a is 50, key is test_row_0/B:col10/1731091263438/Put/seqid=0 2024-11-08T18:41:04,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742135_1311 (size=12301) 2024-11-08T18:41:04,590 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091324589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,591 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091324709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091324709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091324709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:04,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:04,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-08T18:41:04,749 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:04,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:04,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-08T18:41:04,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T18:41:04,751 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:04,752 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:04,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:04,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T18:41:04,890 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/960ad7d324a343b08f27ef6c871f726a 2024-11-08T18:41:04,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/1f81fa5f4d1946ec8e2a55158b6bc2b5 is 50, key is test_row_0/C:col10/1731091263438/Put/seqid=0 2024-11-08T18:41:04,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742136_1312 (size=12301) 2024-11-08T18:41:04,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-08T18:41:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:41:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:04,904 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:04,904 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:05,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-08T18:41:05,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:05,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:41:05,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:05,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:05,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:05,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:05,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T18:41:05,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-08T18:41:05,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:05,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:41:05,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:05,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:05,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:05,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:05,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:05,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091325220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091325220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:05,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091325220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:05,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/1f81fa5f4d1946ec8e2a55158b6bc2b5 2024-11-08T18:41:05,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/f3db5fbc2cd84942b8d1887b610b114a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/f3db5fbc2cd84942b8d1887b610b114a 2024-11-08T18:41:05,308 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/f3db5fbc2cd84942b8d1887b610b114a, entries=150, sequenceid=360, filesize=12.0 K 2024-11-08T18:41:05,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/960ad7d324a343b08f27ef6c871f726a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/960ad7d324a343b08f27ef6c871f726a 2024-11-08T18:41:05,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/960ad7d324a343b08f27ef6c871f726a, entries=150, sequenceid=360, filesize=12.0 K 2024-11-08T18:41:05,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/1f81fa5f4d1946ec8e2a55158b6bc2b5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1f81fa5f4d1946ec8e2a55158b6bc2b5 2024-11-08T18:41:05,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1f81fa5f4d1946ec8e2a55158b6bc2b5, entries=150, sequenceid=360, filesize=12.0 K 2024-11-08T18:41:05,317 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1246ms, sequenceid=360, compaction requested=true 2024-11-08T18:41:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:05,317 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:05,317 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:05,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:05,318 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:05,318 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:05,318 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/A is initiating minor compaction (all files) 2024-11-08T18:41:05,318 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/B is initiating minor compaction (all files) 2024-11-08T18:41:05,318 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/A in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:05,318 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/B in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:05,318 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/b27a15cc8aa14cbbb46cb85fb917562e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5cf588beacc84038876c365dcba504fd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/960ad7d324a343b08f27ef6c871f726a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.7 K 2024-11-08T18:41:05,318 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b0439dfa33074a8ab132a9c5b760ab5b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/41335f1e4467431880a7950c66bf7b51, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/f3db5fbc2cd84942b8d1887b610b114a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.7 K 2024-11-08T18:41:05,319 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0439dfa33074a8ab132a9c5b760ab5b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731091260617 2024-11-08T18:41:05,319 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting b27a15cc8aa14cbbb46cb85fb917562e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731091260617 2024-11-08T18:41:05,319 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cf588beacc84038876c365dcba504fd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731091261252 2024-11-08T18:41:05,319 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41335f1e4467431880a7950c66bf7b51, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731091261252 2024-11-08T18:41:05,319 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 960ad7d324a343b08f27ef6c871f726a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731091263438 2024-11-08T18:41:05,319 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3db5fbc2cd84942b8d1887b610b114a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731091263438 2024-11-08T18:41:05,327 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#B#compaction#267 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:05,328 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/30abd219a3f2429f8901eb39cd8ae958 is 50, key is test_row_0/B:col10/1731091263438/Put/seqid=0 2024-11-08T18:41:05,328 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#A#compaction#268 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:05,329 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/5b42ce2e1a0847d1ab049344d88efb83 is 50, key is test_row_0/A:col10/1731091263438/Put/seqid=0 2024-11-08T18:41:05,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742137_1313 (size=13119) 2024-11-08T18:41:05,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742138_1314 (size=13119) 2024-11-08T18:41:05,352 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/5b42ce2e1a0847d1ab049344d88efb83 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/5b42ce2e1a0847d1ab049344d88efb83 2024-11-08T18:41:05,357 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/A of 0f4cfbb8cd1ae677542dbdc6afe60732 into 5b42ce2e1a0847d1ab049344d88efb83(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:05,357 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:05,357 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/A, priority=13, startTime=1731091265317; duration=0sec 2024-11-08T18:41:05,357 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:05,357 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:A 2024-11-08T18:41:05,357 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:05,358 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:05,358 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/C is initiating minor compaction (all files) 2024-11-08T18:41:05,358 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/C in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:05,358 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/e402e29cee524b18ab593b2e485e029e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/87768a98f2f94c8abad20bfef83cb290, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1f81fa5f4d1946ec8e2a55158b6bc2b5] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.7 K 2024-11-08T18:41:05,358 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e402e29cee524b18ab593b2e485e029e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731091260617 2024-11-08T18:41:05,359 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87768a98f2f94c8abad20bfef83cb290, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1731091261252 2024-11-08T18:41:05,359 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f81fa5f4d1946ec8e2a55158b6bc2b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731091263438 2024-11-08T18:41:05,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-08T18:41:05,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:05,362 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T18:41:05,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/fed6e1b3963845a890e833870762f6e6 is 50, key is test_row_0/A:col10/1731091264072/Put/seqid=0 2024-11-08T18:41:05,369 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#C#compaction#270 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:05,369 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/0bd404c437564c1ca6c58d1929d764ea is 50, key is test_row_0/C:col10/1731091263438/Put/seqid=0 2024-11-08T18:41:05,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T18:41:05,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742139_1315 (size=12301) 2024-11-08T18:41:05,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742140_1316 (size=13119) 2024-11-08T18:41:05,392 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/0bd404c437564c1ca6c58d1929d764ea as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/0bd404c437564c1ca6c58d1929d764ea 2024-11-08T18:41:05,398 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/C of 0f4cfbb8cd1ae677542dbdc6afe60732 into 0bd404c437564c1ca6c58d1929d764ea(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:05,398 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:05,398 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/C, priority=13, startTime=1731091265317; duration=0sec 2024-11-08T18:41:05,398 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:05,398 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:C 2024-11-08T18:41:05,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:05,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. as already flushing 2024-11-08T18:41:05,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:05,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091325699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:05,744 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/30abd219a3f2429f8901eb39cd8ae958 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/30abd219a3f2429f8901eb39cd8ae958 2024-11-08T18:41:05,748 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/B of 0f4cfbb8cd1ae677542dbdc6afe60732 into 30abd219a3f2429f8901eb39cd8ae958(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:05,748 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:05,748 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/B, priority=13, startTime=1731091265317; duration=0sec 2024-11-08T18:41:05,748 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:05,748 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:B 2024-11-08T18:41:05,785 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/fed6e1b3963845a890e833870762f6e6 2024-11-08T18:41:05,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/2393a9ac85f4489aa59e7b27f14c6279 is 50, key is test_row_0/B:col10/1731091264072/Put/seqid=0 2024-11-08T18:41:05,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742141_1317 (size=12301) 2024-11-08T18:41:05,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091325809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:05,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:05,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T18:41:06,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:06,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091326019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,022 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:06,209 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/2393a9ac85f4489aa59e7b27f14c6279 2024-11-08T18:41:06,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/07715960e97e482fbcd213d43f77a95d is 50, key is test_row_0/C:col10/1731091264072/Put/seqid=0 2024-11-08T18:41:06,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742142_1318 (size=12301) 2024-11-08T18:41:06,224 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/07715960e97e482fbcd213d43f77a95d 2024-11-08T18:41:06,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/fed6e1b3963845a890e833870762f6e6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fed6e1b3963845a890e833870762f6e6 2024-11-08T18:41:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36358 deadline: 1731091326230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:06,233 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fed6e1b3963845a890e833870762f6e6, entries=150, sequenceid=369, filesize=12.0 K 2024-11-08T18:41:06,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36350 deadline: 1731091326230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:06,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36284 deadline: 1731091326231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:06,234 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 0f4cfbb8cd1ae677542dbdc6afe60732 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:06,234 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_2' in 0f4cfbb8cd1ae677542dbdc6afe60732 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:06,234 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 0f4cfbb8cd1ae677542dbdc6afe60732 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:06,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:06,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/2393a9ac85f4489aa59e7b27f14c6279 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/2393a9ac85f4489aa59e7b27f14c6279 2024-11-08T18:41:06,240 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/2393a9ac85f4489aa59e7b27f14c6279, entries=150, sequenceid=369, filesize=12.0 K 2024-11-08T18:41:06,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/07715960e97e482fbcd213d43f77a95d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/07715960e97e482fbcd213d43f77a95d 2024-11-08T18:41:06,246 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/07715960e97e482fbcd213d43f77a95d, entries=150, sequenceid=369, filesize=12.0 K 2024-11-08T18:41:06,247 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 885ms, sequenceid=369, compaction requested=false 2024-11-08T18:41:06,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:06,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:06,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-08T18:41:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-08T18:41:06,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-08T18:41:06,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-08T18:41:06,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 1.5020 sec 2024-11-08T18:41:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:06,331 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:06,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-08T18:41:06,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/7e9fd98649c44b3782218001ce3f4d33 is 50, key is test_row_0/A:col10/1731091265653/Put/seqid=0 2024-11-08T18:41:06,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742143_1319 (size=14741) 2024-11-08T18:41:06,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:06,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091326347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:06,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,351 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:06,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091326459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:06,460 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,461 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:06,548 INFO [Thread-1139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1139. 2024-11-08T18:41:06,548 DEBUG [Thread-1139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:06,548 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,548 DEBUG [Thread-1139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,548 INFO [Thread-1135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1135. 2024-11-08T18:41:06,548 DEBUG [Thread-1135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:06,548 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:06,548 DEBUG [Thread-1135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,548 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:06,549 INFO [Thread-1141 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1141. 2024-11-08T18:41:06,549 DEBUG [Thread-1141 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:06,549 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,549 DEBUG [Thread-1141 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,549 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:06,550 INFO [Thread-1133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1133. 2024-11-08T18:41:06,550 DEBUG [Thread-1133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:06,550 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,551 DEBUG [Thread-1133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:06,551 INFO [Thread-1137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1137. 2024-11-08T18:41:06,551 DEBUG [Thread-1137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:06,551 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,551 DEBUG [Thread-1137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:06,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:06,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:06,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091326669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,670 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:06,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/7e9fd98649c44b3782218001ce3f4d33 2024-11-08T18:41:06,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/d241587c1be247569031e0d3d9f4e0c6 is 50, key is test_row_0/B:col10/1731091265653/Put/seqid=0 2024-11-08T18:41:06,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742144_1320 (size=12301) 2024-11-08T18:41:06,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-08T18:41:06,899 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:06,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:06,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091326979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:06,980 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:07,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/d241587c1be247569031e0d3d9f4e0c6 2024-11-08T18:41:07,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/97098d27b4c64f2ab5431c702f003265 is 50, key is test_row_0/C:col10/1731091265653/Put/seqid=0 2024-11-08T18:41:07,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742145_1321 (size=12301) 2024-11-08T18:41:07,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36300 deadline: 1731091327490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:07,490 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:07,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:07,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/97098d27b4c64f2ab5431c702f003265 2024-11-08T18:41:07,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/7e9fd98649c44b3782218001ce3f4d33 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7e9fd98649c44b3782218001ce3f4d33 2024-11-08T18:41:07,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7e9fd98649c44b3782218001ce3f4d33, entries=200, sequenceid=400, filesize=14.4 K 2024-11-08T18:41:07,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/d241587c1be247569031e0d3d9f4e0c6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/d241587c1be247569031e0d3d9f4e0c6 2024-11-08T18:41:07,587 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/d241587c1be247569031e0d3d9f4e0c6, entries=150, sequenceid=400, filesize=12.0 K 2024-11-08T18:41:07,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/97098d27b4c64f2ab5431c702f003265 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/97098d27b4c64f2ab5431c702f003265 2024-11-08T18:41:07,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/97098d27b4c64f2ab5431c702f003265, entries=150, sequenceid=400, filesize=12.0 K 2024-11-08T18:41:07,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=26.84 KB/27480 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1261ms, sequenceid=400, compaction requested=true 2024-11-08T18:41:07,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:07,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:07,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:07,593 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:07,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:07,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:07,593 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:07,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0f4cfbb8cd1ae677542dbdc6afe60732:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:07,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:07,594 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:07,594 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:07,594 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/B is initiating minor compaction (all files) 2024-11-08T18:41:07,594 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/A is initiating minor compaction (all files) 2024-11-08T18:41:07,594 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/A in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:07,594 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/B in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:07,594 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/30abd219a3f2429f8901eb39cd8ae958, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/2393a9ac85f4489aa59e7b27f14c6279, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/d241587c1be247569031e0d3d9f4e0c6] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.8 K 2024-11-08T18:41:07,594 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/5b42ce2e1a0847d1ab049344d88efb83, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fed6e1b3963845a890e833870762f6e6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7e9fd98649c44b3782218001ce3f4d33] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=39.2 K 2024-11-08T18:41:07,595 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b42ce2e1a0847d1ab049344d88efb83, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731091263438 2024-11-08T18:41:07,595 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 30abd219a3f2429f8901eb39cd8ae958, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731091263438 2024-11-08T18:41:07,595 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2393a9ac85f4489aa59e7b27f14c6279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731091264072 2024-11-08T18:41:07,595 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fed6e1b3963845a890e833870762f6e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731091264072 2024-11-08T18:41:07,595 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d241587c1be247569031e0d3d9f4e0c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731091265653 2024-11-08T18:41:07,595 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e9fd98649c44b3782218001ce3f4d33, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731091265653 2024-11-08T18:41:07,604 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#B#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:07,604 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#A#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:07,605 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/fb398c00ff6747309607cf5449c8e446 is 50, key is test_row_0/A:col10/1731091265653/Put/seqid=0 2024-11-08T18:41:07,605 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/ff898659525b4a419cafffa187d92381 is 50, key is test_row_0/B:col10/1731091265653/Put/seqid=0 2024-11-08T18:41:07,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742146_1322 (size=13221) 2024-11-08T18:41:07,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742147_1323 (size=13221) 2024-11-08T18:41:08,020 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/ff898659525b4a419cafffa187d92381 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/ff898659525b4a419cafffa187d92381 2024-11-08T18:41:08,020 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/fb398c00ff6747309607cf5449c8e446 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fb398c00ff6747309607cf5449c8e446 2024-11-08T18:41:08,026 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/B of 0f4cfbb8cd1ae677542dbdc6afe60732 into ff898659525b4a419cafffa187d92381(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:08,026 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/A of 0f4cfbb8cd1ae677542dbdc6afe60732 into fb398c00ff6747309607cf5449c8e446(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:08,026 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:08,026 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:08,026 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/B, priority=13, startTime=1731091267593; duration=0sec 2024-11-08T18:41:08,026 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/A, priority=13, startTime=1731091267593; duration=0sec 2024-11-08T18:41:08,026 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:08,026 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:B 2024-11-08T18:41:08,026 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:08,026 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:A 2024-11-08T18:41:08,027 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:08,028 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:08,028 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 0f4cfbb8cd1ae677542dbdc6afe60732/C is initiating minor compaction (all files) 2024-11-08T18:41:08,028 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0f4cfbb8cd1ae677542dbdc6afe60732/C in TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:08,028 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/0bd404c437564c1ca6c58d1929d764ea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/07715960e97e482fbcd213d43f77a95d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/97098d27b4c64f2ab5431c702f003265] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp, totalSize=36.8 K 2024-11-08T18:41:08,028 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bd404c437564c1ca6c58d1929d764ea, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731091263438 2024-11-08T18:41:08,028 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 07715960e97e482fbcd213d43f77a95d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731091264072 2024-11-08T18:41:08,029 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 97098d27b4c64f2ab5431c702f003265, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=400, earliestPutTs=1731091265653 2024-11-08T18:41:08,037 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0f4cfbb8cd1ae677542dbdc6afe60732#C#compaction#278 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:08,037 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/0019c31eba5d483ebec1ae8f834a8655 is 50, key is test_row_0/C:col10/1731091265653/Put/seqid=0 2024-11-08T18:41:08,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742148_1324 (size=13221) 2024-11-08T18:41:08,245 INFO [Thread-1128 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1128. 2024-11-08T18:41:08,245 DEBUG [Thread-1128 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:08,245 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:08,246 DEBUG [Thread-1128 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:08,246 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:08,251 INFO [Thread-1122 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1122. 2024-11-08T18:41:08,251 DEBUG [Thread-1122 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:08,251 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:08,251 DEBUG [Thread-1122 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:08,251 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:08,261 INFO [Thread-1130 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1130. 2024-11-08T18:41:08,261 DEBUG [Thread-1130 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:08,261 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:08,261 DEBUG [Thread-1130 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:08,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:08,449 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/0019c31eba5d483ebec1ae8f834a8655 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/0019c31eba5d483ebec1ae8f834a8655 2024-11-08T18:41:08,457 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0f4cfbb8cd1ae677542dbdc6afe60732/C of 0f4cfbb8cd1ae677542dbdc6afe60732 into 0019c31eba5d483ebec1ae8f834a8655(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:08,457 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:08,457 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., storeName=0f4cfbb8cd1ae677542dbdc6afe60732/C, priority=13, startTime=1731091267593; duration=0sec 2024-11-08T18:41:08,457 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:08,457 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0f4cfbb8cd1ae677542dbdc6afe60732:C 2024-11-08T18:41:08,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:08,503 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:08,503 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:41:08,503 INFO [Thread-1124 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1124. 2024-11-08T18:41:08,504 DEBUG [Thread-1124 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:08,504 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:08,504 DEBUG [Thread-1124 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:08,504 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:08,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b86abd33ff6349938f28b9b7b25ff5f8 is 50, key is test_row_0/A:col10/1731091268500/Put/seqid=0 2024-11-08T18:41:08,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742149_1325 (size=12301) 2024-11-08T18:41:08,915 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b86abd33ff6349938f28b9b7b25ff5f8 2024-11-08T18:41:08,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/87328c56f15f42e986ac31fc81da9b08 is 50, key is test_row_0/B:col10/1731091268500/Put/seqid=0 2024-11-08T18:41:08,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742150_1326 (size=12301) 2024-11-08T18:41:09,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/87328c56f15f42e986ac31fc81da9b08 2024-11-08T18:41:09,351 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/7a8e67b393774cd9890c365640ac153e is 50, key is test_row_0/C:col10/1731091268500/Put/seqid=0 2024-11-08T18:41:09,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742151_1327 (size=12301) 2024-11-08T18:41:09,756 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/7a8e67b393774cd9890c365640ac153e 2024-11-08T18:41:09,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/b86abd33ff6349938f28b9b7b25ff5f8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b86abd33ff6349938f28b9b7b25ff5f8 2024-11-08T18:41:09,771 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b86abd33ff6349938f28b9b7b25ff5f8, entries=150, sequenceid=414, filesize=12.0 K 2024-11-08T18:41:09,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/87328c56f15f42e986ac31fc81da9b08 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/87328c56f15f42e986ac31fc81da9b08 2024-11-08T18:41:09,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/87328c56f15f42e986ac31fc81da9b08, entries=150, sequenceid=414, filesize=12.0 K 2024-11-08T18:41:09,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/7a8e67b393774cd9890c365640ac153e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7a8e67b393774cd9890c365640ac153e 2024-11-08T18:41:09,781 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7a8e67b393774cd9890c365640ac153e, entries=150, sequenceid=414, filesize=12.0 K 2024-11-08T18:41:09,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=0 B/0 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1279ms, sequenceid=414, compaction requested=false 2024-11-08T18:41:09,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0f4cfbb8cd1ae677542dbdc6afe60732: 2024-11-08T18:41:11,638 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T18:41:12,553 INFO [Thread-1126 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1126. 2024-11-08T18:41:12,553 DEBUG [Thread-1126 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:12,553 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:12,553 DEBUG [Thread-1126 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:12,553 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 60 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 58 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4172 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4744 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4404 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4186 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4769 2024-11-08T18:41:12,554 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T18:41:12,554 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T18:41:12,555 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:41:12,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:12,555 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:12,555 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:12,557 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T18:41:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T18:41:12,563 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091272563"}]},"ts":"1731091272563"} 2024-11-08T18:41:12,565 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T18:41:12,565 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T18:41:12,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:41:12,568 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, UNASSIGN}] 2024-11-08T18:41:12,570 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, UNASSIGN 2024-11-08T18:41:12,571 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0f4cfbb8cd1ae677542dbdc6afe60732, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:12,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, UNASSIGN because future has completed 2024-11-08T18:41:12,573 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:41:12,573 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:41:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T18:41:12,727 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:12,727 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:41:12,727 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 0f4cfbb8cd1ae677542dbdc6afe60732, disabling compactions & flushes 2024-11-08T18:41:12,727 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:12,728 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:12,728 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. after waiting 0 ms 2024-11-08T18:41:12,728 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:12,728 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 0f4cfbb8cd1ae677542dbdc6afe60732 3/3 column families, dataSize=6.71 KB heapSize=18.33 KB 2024-11-08T18:41:12,735 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/cc912a1f133847748c19ee618fb97627 is 50, key is test_row_2/A:col10/1731091272550/Put/seqid=0 2024-11-08T18:41:12,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742152_1328 (size=7415) 2024-11-08T18:41:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T18:41:13,143 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/cc912a1f133847748c19ee618fb97627 2024-11-08T18:41:13,157 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/68021f0db7004b8e98282cdf622501c5 is 50, key is test_row_2/B:col10/1731091272550/Put/seqid=0 2024-11-08T18:41:13,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742153_1329 (size=7415) 2024-11-08T18:41:13,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T18:41:13,562 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/68021f0db7004b8e98282cdf622501c5 2024-11-08T18:41:13,577 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/671aeacae7f34910b80098ce1eff1ddd is 50, key is test_row_2/C:col10/1731091272550/Put/seqid=0 2024-11-08T18:41:13,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742154_1330 (size=7415) 2024-11-08T18:41:13,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T18:41:13,982 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.24 KB at sequenceid=418 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/671aeacae7f34910b80098ce1eff1ddd 2024-11-08T18:41:13,992 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/A/cc912a1f133847748c19ee618fb97627 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/cc912a1f133847748c19ee618fb97627 2024-11-08T18:41:13,998 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/cc912a1f133847748c19ee618fb97627, entries=50, sequenceid=418, filesize=7.2 K 2024-11-08T18:41:13,999 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/B/68021f0db7004b8e98282cdf622501c5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/68021f0db7004b8e98282cdf622501c5 2024-11-08T18:41:14,003 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/68021f0db7004b8e98282cdf622501c5, entries=50, sequenceid=418, filesize=7.2 K 2024-11-08T18:41:14,003 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/.tmp/C/671aeacae7f34910b80098ce1eff1ddd as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/671aeacae7f34910b80098ce1eff1ddd 2024-11-08T18:41:14,007 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/671aeacae7f34910b80098ce1eff1ddd, entries=50, sequenceid=418, filesize=7.2 K 2024-11-08T18:41:14,008 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1280ms, sequenceid=418, compaction requested=true 2024-11-08T18:41:14,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/2872a575d8824aeebb870dcebaa5721d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/4f29c4d4f40643389afd51b16080bfd3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/226002f8e9584399a87b48c8441a80f6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f742a80b9f54d78b5e5ee39de41e0cb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7f311f474dc1446aa16e2a09649424ba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/aef37567f59a40ad96ea44ce9c1aa991, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/c1164a9bcdc84ae5a5bb5ecd1709b244, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/16172339345e4450881e135f916f0ca3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f035b63a6704a45af876372a4365884, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b76ab2a215cc4517a37f637af09083cd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b802d0325cce4ec18d89e363ea8a7387, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/ed3ef75018464809a32cdea6001e4291, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/a0954fa8fefb43ca852883bf56b17546, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7cebe61155714204ba1f29006ed7ae91, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b33d02daa84145e5b6f98006ce08096a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/90f89a2523e343fa890fdf755d17dbe1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/94ae3c6a5e364b839ad51efe9f3a873c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/25900563c61b47119afb7ab5bf599aba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/15299a4ec9054f338ceaf1aa05de4d61, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/d1acb0eed3a04d41a9ae543f6904a4ee, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b0439dfa33074a8ab132a9c5b760ab5b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/46879505afbc4921b195776ded81d01b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/41335f1e4467431880a7950c66bf7b51, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/5b42ce2e1a0847d1ab049344d88efb83, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/f3db5fbc2cd84942b8d1887b610b114a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fed6e1b3963845a890e833870762f6e6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7e9fd98649c44b3782218001ce3f4d33] to archive 2024-11-08T18:41:14,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:41:14,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/2872a575d8824aeebb870dcebaa5721d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/2872a575d8824aeebb870dcebaa5721d 2024-11-08T18:41:14,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/4f29c4d4f40643389afd51b16080bfd3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/4f29c4d4f40643389afd51b16080bfd3 2024-11-08T18:41:14,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/226002f8e9584399a87b48c8441a80f6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/226002f8e9584399a87b48c8441a80f6 2024-11-08T18:41:14,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f742a80b9f54d78b5e5ee39de41e0cb to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f742a80b9f54d78b5e5ee39de41e0cb 2024-11-08T18:41:14,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7f311f474dc1446aa16e2a09649424ba to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7f311f474dc1446aa16e2a09649424ba 2024-11-08T18:41:14,018 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/aef37567f59a40ad96ea44ce9c1aa991 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/aef37567f59a40ad96ea44ce9c1aa991 2024-11-08T18:41:14,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/c1164a9bcdc84ae5a5bb5ecd1709b244 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/c1164a9bcdc84ae5a5bb5ecd1709b244 2024-11-08T18:41:14,020 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/16172339345e4450881e135f916f0ca3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/16172339345e4450881e135f916f0ca3 2024-11-08T18:41:14,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f035b63a6704a45af876372a4365884 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/8f035b63a6704a45af876372a4365884 2024-11-08T18:41:14,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b76ab2a215cc4517a37f637af09083cd to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b76ab2a215cc4517a37f637af09083cd 2024-11-08T18:41:14,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b802d0325cce4ec18d89e363ea8a7387 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b802d0325cce4ec18d89e363ea8a7387 2024-11-08T18:41:14,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/ed3ef75018464809a32cdea6001e4291 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/ed3ef75018464809a32cdea6001e4291 2024-11-08T18:41:14,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/a0954fa8fefb43ca852883bf56b17546 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/a0954fa8fefb43ca852883bf56b17546 2024-11-08T18:41:14,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7cebe61155714204ba1f29006ed7ae91 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7cebe61155714204ba1f29006ed7ae91 2024-11-08T18:41:14,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b33d02daa84145e5b6f98006ce08096a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b33d02daa84145e5b6f98006ce08096a 2024-11-08T18:41:14,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/90f89a2523e343fa890fdf755d17dbe1 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/90f89a2523e343fa890fdf755d17dbe1 2024-11-08T18:41:14,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/94ae3c6a5e364b839ad51efe9f3a873c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/94ae3c6a5e364b839ad51efe9f3a873c 2024-11-08T18:41:14,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/25900563c61b47119afb7ab5bf599aba to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/25900563c61b47119afb7ab5bf599aba 2024-11-08T18:41:14,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/15299a4ec9054f338ceaf1aa05de4d61 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/15299a4ec9054f338ceaf1aa05de4d61 2024-11-08T18:41:14,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/d1acb0eed3a04d41a9ae543f6904a4ee to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/d1acb0eed3a04d41a9ae543f6904a4ee 2024-11-08T18:41:14,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b0439dfa33074a8ab132a9c5b760ab5b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b0439dfa33074a8ab132a9c5b760ab5b 2024-11-08T18:41:14,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/46879505afbc4921b195776ded81d01b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/46879505afbc4921b195776ded81d01b 2024-11-08T18:41:14,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/41335f1e4467431880a7950c66bf7b51 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/41335f1e4467431880a7950c66bf7b51 2024-11-08T18:41:14,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/5b42ce2e1a0847d1ab049344d88efb83 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/5b42ce2e1a0847d1ab049344d88efb83 2024-11-08T18:41:14,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/f3db5fbc2cd84942b8d1887b610b114a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/f3db5fbc2cd84942b8d1887b610b114a 2024-11-08T18:41:14,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fed6e1b3963845a890e833870762f6e6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fed6e1b3963845a890e833870762f6e6 2024-11-08T18:41:14,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7e9fd98649c44b3782218001ce3f4d33 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/7e9fd98649c44b3782218001ce3f4d33 2024-11-08T18:41:14,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/6a91d243250d48778214cd473d9681e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/35888a5f095846abb20741b916d77044, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/da55f1897f7043b19862e7166c97d875, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/45857c049dbd45b5abe037ed32a5206f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5858cf5b1f8b404a990744a513194713, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f0777622835c485887581ca37066e4b9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/0aa6a7a76d96494aaf711c3cdf5562f2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/28742d4edf274318b5551b3162f154fa, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/636eaba890554018ab53d1d6b1f6a316, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5eff674b78184d5c88d5b3bfe7652028, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01ded053bef447a68fecee8dfaed815d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01711d79c77c44389e6f62a72e64b610, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f7aebca2107e43e0b0a67dbdcea55faf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/a6dceee991ff43cfaa528985f7ffd5c0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f5c1d581448a4628bf37aa9ca0c4d59d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/4d47b70f4cf14e129da92c296be26166, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/eb5269a210864d5782e249fb31222036, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/675bd3bc2bef4207aadf67c67ee4fb46, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f23f33a6e5e847e39326e3d530a5ea35, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/076235b2d3854793b07eb0d9635e4db0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/b27a15cc8aa14cbbb46cb85fb917562e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/dcc217c1f2cd414eaf9a8a4b8647e781, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5cf588beacc84038876c365dcba504fd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/30abd219a3f2429f8901eb39cd8ae958, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/960ad7d324a343b08f27ef6c871f726a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/2393a9ac85f4489aa59e7b27f14c6279, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/d241587c1be247569031e0d3d9f4e0c6] to archive 2024-11-08T18:41:14,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:41:14,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/6a91d243250d48778214cd473d9681e3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/6a91d243250d48778214cd473d9681e3 2024-11-08T18:41:14,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/35888a5f095846abb20741b916d77044 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/35888a5f095846abb20741b916d77044 2024-11-08T18:41:14,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/da55f1897f7043b19862e7166c97d875 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/da55f1897f7043b19862e7166c97d875 2024-11-08T18:41:14,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/45857c049dbd45b5abe037ed32a5206f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/45857c049dbd45b5abe037ed32a5206f 2024-11-08T18:41:14,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5858cf5b1f8b404a990744a513194713 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5858cf5b1f8b404a990744a513194713 2024-11-08T18:41:14,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f0777622835c485887581ca37066e4b9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f0777622835c485887581ca37066e4b9 2024-11-08T18:41:14,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/0aa6a7a76d96494aaf711c3cdf5562f2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/0aa6a7a76d96494aaf711c3cdf5562f2 2024-11-08T18:41:14,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/28742d4edf274318b5551b3162f154fa to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/28742d4edf274318b5551b3162f154fa 2024-11-08T18:41:14,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/636eaba890554018ab53d1d6b1f6a316 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/636eaba890554018ab53d1d6b1f6a316 2024-11-08T18:41:14,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5eff674b78184d5c88d5b3bfe7652028 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5eff674b78184d5c88d5b3bfe7652028 2024-11-08T18:41:14,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01ded053bef447a68fecee8dfaed815d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01ded053bef447a68fecee8dfaed815d 2024-11-08T18:41:14,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01711d79c77c44389e6f62a72e64b610 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/01711d79c77c44389e6f62a72e64b610 2024-11-08T18:41:14,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f7aebca2107e43e0b0a67dbdcea55faf to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f7aebca2107e43e0b0a67dbdcea55faf 2024-11-08T18:41:14,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/a6dceee991ff43cfaa528985f7ffd5c0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/a6dceee991ff43cfaa528985f7ffd5c0 2024-11-08T18:41:14,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f5c1d581448a4628bf37aa9ca0c4d59d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f5c1d581448a4628bf37aa9ca0c4d59d 2024-11-08T18:41:14,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/4d47b70f4cf14e129da92c296be26166 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/4d47b70f4cf14e129da92c296be26166 2024-11-08T18:41:14,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/eb5269a210864d5782e249fb31222036 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/eb5269a210864d5782e249fb31222036 2024-11-08T18:41:14,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/675bd3bc2bef4207aadf67c67ee4fb46 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/675bd3bc2bef4207aadf67c67ee4fb46 2024-11-08T18:41:14,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f23f33a6e5e847e39326e3d530a5ea35 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/f23f33a6e5e847e39326e3d530a5ea35 2024-11-08T18:41:14,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/076235b2d3854793b07eb0d9635e4db0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/076235b2d3854793b07eb0d9635e4db0 2024-11-08T18:41:14,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/b27a15cc8aa14cbbb46cb85fb917562e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/b27a15cc8aa14cbbb46cb85fb917562e 2024-11-08T18:41:14,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/dcc217c1f2cd414eaf9a8a4b8647e781 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/dcc217c1f2cd414eaf9a8a4b8647e781 2024-11-08T18:41:14,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5cf588beacc84038876c365dcba504fd to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/5cf588beacc84038876c365dcba504fd 2024-11-08T18:41:14,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/30abd219a3f2429f8901eb39cd8ae958 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/30abd219a3f2429f8901eb39cd8ae958 2024-11-08T18:41:14,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/960ad7d324a343b08f27ef6c871f726a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/960ad7d324a343b08f27ef6c871f726a 2024-11-08T18:41:14,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/2393a9ac85f4489aa59e7b27f14c6279 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/2393a9ac85f4489aa59e7b27f14c6279 2024-11-08T18:41:14,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/d241587c1be247569031e0d3d9f4e0c6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/d241587c1be247569031e0d3d9f4e0c6 2024-11-08T18:41:14,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/219c1c7023724d2fade01a4d5291b3b4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9fb845530f5249569e5cbe8f36954029, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/f5a1135446794516a0128c8080be3d06, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/8e975f89a50044a889e7f485788a1997, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9c2730865b4146afabf1b5c36431394c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/84fb094e46fb43ce83eae0844a7879d9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/aec091f62483435197d8e8c1edee0797, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7102010f84314198bedb85b1b0ef1298, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/54c701aeaadc4f21b4bd7b4a6dd1691f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/c578f654fdf84a68ba1654359d161d7a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cadf38077f6243fa903f2d568d3f7aae, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/2b2d0f3f9a97458699a97ae79bbde161, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/09dca93440c7422f9b36ba7b1257c5fc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/16735ce68ab74b08acafd43ec67f7e8c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/70c1c3c639a94c5bb65cf1903e74238a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/be779b46a07a43769259436320522f80, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cef7be2dfd7c4c7ea4eac3c741d3768f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/374f3b37544348bc80b9fff298b3b48e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/ee63f62f22e94ec49b6bfbd216a224ea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1602f9fe4df144dd82bf6e25d04956cb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/e402e29cee524b18ab593b2e485e029e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/009a203c24f7438b95e5cd3aec3e0db8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/87768a98f2f94c8abad20bfef83cb290, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/0bd404c437564c1ca6c58d1929d764ea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1f81fa5f4d1946ec8e2a55158b6bc2b5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/07715960e97e482fbcd213d43f77a95d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/97098d27b4c64f2ab5431c702f003265] to archive 2024-11-08T18:41:14,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:41:14,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/219c1c7023724d2fade01a4d5291b3b4 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/219c1c7023724d2fade01a4d5291b3b4 2024-11-08T18:41:14,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9fb845530f5249569e5cbe8f36954029 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9fb845530f5249569e5cbe8f36954029 2024-11-08T18:41:14,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/f5a1135446794516a0128c8080be3d06 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/f5a1135446794516a0128c8080be3d06 2024-11-08T18:41:14,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/8e975f89a50044a889e7f485788a1997 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/8e975f89a50044a889e7f485788a1997 2024-11-08T18:41:14,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9c2730865b4146afabf1b5c36431394c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/9c2730865b4146afabf1b5c36431394c 2024-11-08T18:41:14,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/84fb094e46fb43ce83eae0844a7879d9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/84fb094e46fb43ce83eae0844a7879d9 2024-11-08T18:41:14,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/aec091f62483435197d8e8c1edee0797 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/aec091f62483435197d8e8c1edee0797 2024-11-08T18:41:14,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7102010f84314198bedb85b1b0ef1298 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7102010f84314198bedb85b1b0ef1298 2024-11-08T18:41:14,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/54c701aeaadc4f21b4bd7b4a6dd1691f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/54c701aeaadc4f21b4bd7b4a6dd1691f 2024-11-08T18:41:14,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/c578f654fdf84a68ba1654359d161d7a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/c578f654fdf84a68ba1654359d161d7a 2024-11-08T18:41:14,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cadf38077f6243fa903f2d568d3f7aae to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cadf38077f6243fa903f2d568d3f7aae 2024-11-08T18:41:14,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/2b2d0f3f9a97458699a97ae79bbde161 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/2b2d0f3f9a97458699a97ae79bbde161 2024-11-08T18:41:14,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/09dca93440c7422f9b36ba7b1257c5fc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/09dca93440c7422f9b36ba7b1257c5fc 2024-11-08T18:41:14,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/16735ce68ab74b08acafd43ec67f7e8c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/16735ce68ab74b08acafd43ec67f7e8c 2024-11-08T18:41:14,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/70c1c3c639a94c5bb65cf1903e74238a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/70c1c3c639a94c5bb65cf1903e74238a 2024-11-08T18:41:14,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/be779b46a07a43769259436320522f80 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/be779b46a07a43769259436320522f80 2024-11-08T18:41:14,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cef7be2dfd7c4c7ea4eac3c741d3768f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/cef7be2dfd7c4c7ea4eac3c741d3768f 2024-11-08T18:41:14,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/374f3b37544348bc80b9fff298b3b48e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/374f3b37544348bc80b9fff298b3b48e 2024-11-08T18:41:14,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/ee63f62f22e94ec49b6bfbd216a224ea to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/ee63f62f22e94ec49b6bfbd216a224ea 2024-11-08T18:41:14,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1602f9fe4df144dd82bf6e25d04956cb to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1602f9fe4df144dd82bf6e25d04956cb 2024-11-08T18:41:14,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/e402e29cee524b18ab593b2e485e029e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/e402e29cee524b18ab593b2e485e029e 2024-11-08T18:41:14,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/009a203c24f7438b95e5cd3aec3e0db8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/009a203c24f7438b95e5cd3aec3e0db8 2024-11-08T18:41:14,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/87768a98f2f94c8abad20bfef83cb290 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/87768a98f2f94c8abad20bfef83cb290 2024-11-08T18:41:14,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/0bd404c437564c1ca6c58d1929d764ea to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/0bd404c437564c1ca6c58d1929d764ea 2024-11-08T18:41:14,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1f81fa5f4d1946ec8e2a55158b6bc2b5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/1f81fa5f4d1946ec8e2a55158b6bc2b5 2024-11-08T18:41:14,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/07715960e97e482fbcd213d43f77a95d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/07715960e97e482fbcd213d43f77a95d 2024-11-08T18:41:14,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/97098d27b4c64f2ab5431c702f003265 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/97098d27b4c64f2ab5431c702f003265 2024-11-08T18:41:14,078 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/recovered.edits/421.seqid, newMaxSeqId=421, maxSeqId=1 2024-11-08T18:41:14,078 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. 2024-11-08T18:41:14,078 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 0f4cfbb8cd1ae677542dbdc6afe60732: Waiting for close lock at 1731091272727Running coprocessor pre-close hooks at 1731091272727Disabling compacts and flushes for region at 1731091272727Disabling writes for close at 1731091272728 (+1 ms)Obtaining lock to block concurrent updates at 1731091272728Preparing flush snapshotting stores in 0f4cfbb8cd1ae677542dbdc6afe60732 at 1731091272728Finished memstore snapshotting TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732., syncing WAL and waiting on mvcc, flushsize=dataSize=6870, getHeapSize=18720, getOffHeapSize=0, getCellsCount=150 at 1731091272729 (+1 ms)Flushing stores of TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732. at 1731091272730 (+1 ms)Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/A: creating writer at 1731091272730Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/A: appending metadata at 1731091272734 (+4 ms)Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/A: closing flushed file at 1731091272735 (+1 ms)Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/B: creating writer at 1731091273154 (+419 ms)Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/B: appending metadata at 1731091273156 (+2 ms)Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/B: closing flushed file at 1731091273156Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/C: creating writer at 1731091273574 (+418 ms)Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/C: appending metadata at 1731091273576 (+2 ms)Flushing 0f4cfbb8cd1ae677542dbdc6afe60732/C: closing flushed file at 1731091273576Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e8e1cc5: reopening flushed file at 1731091273991 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c6f842c: reopening flushed file at 1731091273998 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22c9d68e: reopening flushed file at 1731091274003 (+5 ms)Finished flush of dataSize ~6.71 KB/6870, heapSize ~18.28 KB/18720, currentSize=0 B/0 for 0f4cfbb8cd1ae677542dbdc6afe60732 in 1280ms, sequenceid=418, compaction requested=true at 1731091274008 (+5 ms)Writing region close event to WAL at 1731091274075 (+67 ms)Running coprocessor post-close hooks at 1731091274078 (+3 ms)Closed at 1731091274078 2024-11-08T18:41:14,080 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:14,080 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=0f4cfbb8cd1ae677542dbdc6afe60732, regionState=CLOSED 2024-11-08T18:41:14,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:41:14,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-08T18:41:14,083 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 0f4cfbb8cd1ae677542dbdc6afe60732, server=b75fe890c219,44447,1731091183487 in 1.5090 sec 2024-11-08T18:41:14,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-08T18:41:14,085 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0f4cfbb8cd1ae677542dbdc6afe60732, UNASSIGN in 1.5150 sec 2024-11-08T18:41:14,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-08T18:41:14,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5190 sec 2024-11-08T18:41:14,088 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091274087"}]},"ts":"1731091274087"} 2024-11-08T18:41:14,089 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T18:41:14,089 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T18:41:14,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5320 sec 2024-11-08T18:41:14,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-08T18:41:14,721 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:14,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T18:41:14,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:14,728 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:14,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-08T18:41:14,731 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:14,736 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:14,740 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/recovered.edits] 2024-11-08T18:41:14,743 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b86abd33ff6349938f28b9b7b25ff5f8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/b86abd33ff6349938f28b9b7b25ff5f8 2024-11-08T18:41:14,744 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/cc912a1f133847748c19ee618fb97627 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/cc912a1f133847748c19ee618fb97627 2024-11-08T18:41:14,745 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fb398c00ff6747309607cf5449c8e446 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/A/fb398c00ff6747309607cf5449c8e446 2024-11-08T18:41:14,748 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/68021f0db7004b8e98282cdf622501c5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/68021f0db7004b8e98282cdf622501c5 2024-11-08T18:41:14,749 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/87328c56f15f42e986ac31fc81da9b08 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/87328c56f15f42e986ac31fc81da9b08 2024-11-08T18:41:14,751 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/ff898659525b4a419cafffa187d92381 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/B/ff898659525b4a419cafffa187d92381 2024-11-08T18:41:14,754 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/0019c31eba5d483ebec1ae8f834a8655 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/0019c31eba5d483ebec1ae8f834a8655 2024-11-08T18:41:14,755 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/671aeacae7f34910b80098ce1eff1ddd to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/671aeacae7f34910b80098ce1eff1ddd 2024-11-08T18:41:14,757 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7a8e67b393774cd9890c365640ac153e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/C/7a8e67b393774cd9890c365640ac153e 2024-11-08T18:41:14,760 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/recovered.edits/421.seqid to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732/recovered.edits/421.seqid 2024-11-08T18:41:14,761 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/0f4cfbb8cd1ae677542dbdc6afe60732 2024-11-08T18:41:14,761 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T18:41:14,764 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:14,770 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T18:41:14,773 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T18:41:14,774 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:14,774 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T18:41:14,775 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731091274775"}]},"ts":"9223372036854775807"} 2024-11-08T18:41:14,777 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T18:41:14,777 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0f4cfbb8cd1ae677542dbdc6afe60732, NAME => 'TestAcidGuarantees,,1731091245347.0f4cfbb8cd1ae677542dbdc6afe60732.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T18:41:14,777 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T18:41:14,778 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731091274778"}]},"ts":"9223372036854775807"} 2024-11-08T18:41:14,780 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T18:41:14,780 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:14,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-08T18:41:14,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-08T18:41:14,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T18:41:14,840 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:14,857 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=239 (was 242), OpenFileDescriptor=455 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=407 (was 524), ProcessCount=11 (was 11), AvailableMemoryMB=3769 (was 3892) 2024-11-08T18:41:14,867 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=239, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=407, ProcessCount=11, AvailableMemoryMB=3769 2024-11-08T18:41:14,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:41:14,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:41:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:14,872 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T18:41:14,872 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:14,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-08T18:41:14,873 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T18:41:14,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-08T18:41:14,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742155_1331 (size=959) 2024-11-08T18:41:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-08T18:41:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-08T18:41:15,286 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 2024-11-08T18:41:15,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742156_1332 (size=53) 2024-11-08T18:41:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-08T18:41:15,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:41:15,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2d8b2888d7998b4dc696cf695019721f, disabling compactions & flushes 2024-11-08T18:41:15,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:15,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:15,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. after waiting 0 ms 2024-11-08T18:41:15,698 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:15,699 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:15,699 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2d8b2888d7998b4dc696cf695019721f: Waiting for close lock at 1731091275698Disabling compacts and flushes for region at 1731091275698Disabling writes for close at 1731091275698Writing region close event to WAL at 1731091275698Closed at 1731091275698 2024-11-08T18:41:15,701 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T18:41:15,702 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731091275702"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731091275702"}]},"ts":"1731091275702"} 2024-11-08T18:41:15,706 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T18:41:15,707 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T18:41:15,707 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091275707"}]},"ts":"1731091275707"} 2024-11-08T18:41:15,709 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T18:41:15,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, ASSIGN}] 2024-11-08T18:41:15,710 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, ASSIGN 2024-11-08T18:41:15,711 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, ASSIGN; state=OFFLINE, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=false 2024-11-08T18:41:15,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=2d8b2888d7998b4dc696cf695019721f, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:15,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, ASSIGN because future has completed 2024-11-08T18:41:15,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:41:16,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-08T18:41:16,025 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:16,026 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:41:16,026 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,026 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:41:16,027 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,027 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,029 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,032 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8b2888d7998b4dc696cf695019721f columnFamilyName A 2024-11-08T18:41:16,032 DEBUG [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:16,033 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(327): Store=2d8b2888d7998b4dc696cf695019721f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:16,033 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,035 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8b2888d7998b4dc696cf695019721f columnFamilyName B 2024-11-08T18:41:16,035 DEBUG [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:16,035 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(327): Store=2d8b2888d7998b4dc696cf695019721f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:16,035 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,036 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8b2888d7998b4dc696cf695019721f columnFamilyName C 2024-11-08T18:41:16,036 DEBUG [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:16,037 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(327): Store=2d8b2888d7998b4dc696cf695019721f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:16,037 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,037 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,038 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,039 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,039 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,039 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:41:16,041 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,043 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T18:41:16,043 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 2d8b2888d7998b4dc696cf695019721f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75268589, jitterRate=0.12158937752246857}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:41:16,043 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:16,044 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 2d8b2888d7998b4dc696cf695019721f: Running coprocessor pre-open hook at 1731091276027Writing region info on filesystem at 1731091276027Initializing all the Stores at 1731091276028 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091276028Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091276029 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091276029Cleaning up temporary data from old regions at 1731091276039 (+10 ms)Running coprocessor post-open hooks at 1731091276043 (+4 ms)Region opened successfully at 1731091276044 (+1 ms) 2024-11-08T18:41:16,044 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., pid=89, masterSystemTime=1731091276021 2024-11-08T18:41:16,046 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:16,046 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:16,047 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=2d8b2888d7998b4dc696cf695019721f, regionState=OPEN, openSeqNum=2, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:16,048 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:41:16,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-08T18:41:16,051 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 in 183 msec 2024-11-08T18:41:16,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-08T18:41:16,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, ASSIGN in 342 msec 2024-11-08T18:41:16,054 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T18:41:16,054 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091276054"}]},"ts":"1731091276054"} 2024-11-08T18:41:16,056 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T18:41:16,057 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T18:41:16,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-08T18:41:17,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-08T18:41:17,031 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:17,034 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7db9998b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:17,034 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:17,034 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:17,036 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:17,036 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@375c6607, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:17,037 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:17,038 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:17,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a07e107, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:17,040 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:17,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:17,043 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:17,044 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T18:41:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is b75fe890c219,42293,1731091182659 2024-11-08T18:41:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@757c864b 2024-11-08T18:41:17,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T18:41:17,046 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T18:41:17,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:41:17,048 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:41:17,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:17,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742157_1333 (size=995) 2024-11-08T18:41:17,468 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T18:41:17,468 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T18:41:17,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:41:17,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, REOPEN/MOVE}] 2024-11-08T18:41:17,477 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, REOPEN/MOVE 2024-11-08T18:41:17,478 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2d8b2888d7998b4dc696cf695019721f, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:17,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, REOPEN/MOVE because future has completed 2024-11-08T18:41:17,479 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:41:17,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:41:17,633 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,633 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:41:17,633 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 2d8b2888d7998b4dc696cf695019721f, disabling compactions & flushes 2024-11-08T18:41:17,633 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:17,633 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:17,633 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. after waiting 0 ms 2024-11-08T18:41:17,633 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:17,641 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T18:41:17,642 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:17,643 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 2d8b2888d7998b4dc696cf695019721f: Waiting for close lock at 1731091277633Running coprocessor pre-close hooks at 1731091277633Disabling compacts and flushes for region at 1731091277633Disabling writes for close at 1731091277633Writing region close event to WAL at 1731091277634 (+1 ms)Running coprocessor post-close hooks at 1731091277642 (+8 ms)Closed at 1731091277642 2024-11-08T18:41:17,643 WARN [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 2d8b2888d7998b4dc696cf695019721f to self. 2024-11-08T18:41:17,646 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,647 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2d8b2888d7998b4dc696cf695019721f, regionState=CLOSED 2024-11-08T18:41:17,649 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:41:17,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-08T18:41:17,652 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 in 171 msec 2024-11-08T18:41:17,653 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, REOPEN/MOVE; state=CLOSED, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=true 2024-11-08T18:41:17,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2d8b2888d7998b4dc696cf695019721f, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:17,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, REOPEN/MOVE because future has completed 2024-11-08T18:41:17,809 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:41:17,971 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:17,971 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:41:17,972 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,972 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:41:17,972 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,972 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,974 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,975 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8b2888d7998b4dc696cf695019721f columnFamilyName A 2024-11-08T18:41:17,977 DEBUG [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:17,978 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(327): Store=2d8b2888d7998b4dc696cf695019721f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:17,978 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,979 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8b2888d7998b4dc696cf695019721f columnFamilyName B 2024-11-08T18:41:17,979 DEBUG [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:17,980 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(327): Store=2d8b2888d7998b4dc696cf695019721f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:17,980 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,981 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2d8b2888d7998b4dc696cf695019721f columnFamilyName C 2024-11-08T18:41:17,981 DEBUG [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:17,981 INFO [StoreOpener-2d8b2888d7998b4dc696cf695019721f-1 {}] regionserver.HStore(327): Store=2d8b2888d7998b4dc696cf695019721f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:17,981 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,982 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,984 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,985 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,985 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,986 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:41:17,987 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,988 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 2d8b2888d7998b4dc696cf695019721f; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66641993, jitterRate=-0.006956920027732849}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:41:17,989 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:17,989 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 2d8b2888d7998b4dc696cf695019721f: Running coprocessor pre-open hook at 1731091277972Writing region info on filesystem at 1731091277972Initializing all the Stores at 1731091277973 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091277974 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091277974Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091277974Cleaning up temporary data from old regions at 1731091277985 (+11 ms)Running coprocessor post-open hooks at 1731091277989 (+4 ms)Region opened successfully at 1731091277989 2024-11-08T18:41:17,990 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., pid=94, masterSystemTime=1731091277964 2024-11-08T18:41:17,993 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:17,993 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:17,993 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=2d8b2888d7998b4dc696cf695019721f, regionState=OPEN, openSeqNum=5, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:17,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:41:17,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-08T18:41:17,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 in 187 msec 2024-11-08T18:41:18,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-08T18:41:18,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, REOPEN/MOVE in 523 msec 2024-11-08T18:41:18,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-08T18:41:18,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 528 msec 2024-11-08T18:41:18,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 956 msec 2024-11-08T18:41:18,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-08T18:41:18,007 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:18,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0f1e03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26034ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49655b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,011 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,011 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,012 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e838a68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,013 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,013 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bb527f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,015 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35ce2c11, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,017 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33388, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,017 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14691392, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,019 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7daaffa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,020 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33412, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e74edb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,022 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20d29420, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,022 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,023 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6fd3d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b758c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,025 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,026 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788e4146, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,027 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33442, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20988bc2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53f9edfd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,029 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e6db39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,030 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,031 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,031 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b31ddc9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,032 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42125f21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,032 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,032 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,033 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5d0907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,034 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,035 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5f9d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,036 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64b126a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,036 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,036 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,036 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66c45c9a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,037 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,038 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33484, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f8f8ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,039 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa71035, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,039 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:18,039 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:18,040 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:18,040 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b516408, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:18,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:18,042 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:18,042 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35a6c81a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:18,043 DEBUG [Thread-1539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,043 DEBUG [Thread-1535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,043 DEBUG [Thread-1541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,043 DEBUG [Thread-1524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,043 DEBUG [Thread-1533 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,043 DEBUG [Thread-1526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,044 DEBUG [Thread-1522 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,044 DEBUG [Thread-1541 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,044 DEBUG [Thread-1541 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,045 DEBUG [Thread-1528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,045 DEBUG [Thread-1530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,045 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:18,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36208, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36206, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36202, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36218, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36224, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,046 DEBUG [Thread-1537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:18,046 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,047 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36246, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,048 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-08T18:41:18,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T18:41:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:18,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:18,051 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:18,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:18,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,052 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:18,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:18,052 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,055 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:41:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:18,062 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:18,062 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:41:18,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088410219160ac43a9bbc66212c7134354_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091278058/Put/seqid=0 2024-11-08T18:41:18,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742158_1334 (size=12154) 2024-11-08T18:41:18,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091338122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091338125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091338126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091338130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,134 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091338131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T18:41:18,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T18:41:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091338239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091338240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091338240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091338240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,243 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091338249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,255 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T18:41:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T18:41:18,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091338449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091338449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,454 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091338449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091338449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091338459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:18,493 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088410219160ac43a9bbc66212c7134354_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088410219160ac43a9bbc66212c7134354_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:18,494 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/64048c569a1d442f9c94127f09b1dd46, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:18,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/64048c569a1d442f9c94127f09b1dd46 is 175, key is test_row_0/A:col10/1731091278058/Put/seqid=0 2024-11-08T18:41:18,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742159_1335 (size=30955) 2024-11-08T18:41:18,500 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/64048c569a1d442f9c94127f09b1dd46 2024-11-08T18:41:18,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T18:41:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/1745800b2d6e40b9b7436bdf77454a61 is 50, key is test_row_0/B:col10/1731091278058/Put/seqid=0 2024-11-08T18:41:18,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742160_1336 (size=12001) 2024-11-08T18:41:18,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/1745800b2d6e40b9b7436bdf77454a61 2024-11-08T18:41:18,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/7ebf9f9c3291475a80ac993d5b1ae7c5 is 50, key is test_row_0/C:col10/1731091278058/Put/seqid=0 2024-11-08T18:41:18,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742161_1337 (size=12001) 2024-11-08T18:41:18,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T18:41:18,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:18,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T18:41:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,763 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091338759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:18,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091338770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:18,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T18:41:18,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:18,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/7ebf9f9c3291475a80ac993d5b1ae7c5 2024-11-08T18:41:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/64048c569a1d442f9c94127f09b1dd46 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/64048c569a1d442f9c94127f09b1dd46 2024-11-08T18:41:18,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/64048c569a1d442f9c94127f09b1dd46, entries=150, sequenceid=15, filesize=30.2 K 2024-11-08T18:41:18,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/1745800b2d6e40b9b7436bdf77454a61 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/1745800b2d6e40b9b7436bdf77454a61 2024-11-08T18:41:18,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/1745800b2d6e40b9b7436bdf77454a61, entries=150, sequenceid=15, filesize=11.7 K 2024-11-08T18:41:18,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/7ebf9f9c3291475a80ac993d5b1ae7c5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/7ebf9f9c3291475a80ac993d5b1ae7c5 2024-11-08T18:41:18,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T18:41:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:18,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:18,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/7ebf9f9c3291475a80ac993d5b1ae7c5, entries=150, sequenceid=15, filesize=11.7 K 2024-11-08T18:41:18,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2d8b2888d7998b4dc696cf695019721f in 913ms, sequenceid=15, compaction requested=false 2024-11-08T18:41:18,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:19,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-08T18:41:19,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:19,123 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T18:41:19,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108763850b2da554e7fb574927b980bf9f8_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091278130/Put/seqid=0 2024-11-08T18:41:19,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742162_1338 (size=12154) 2024-11-08T18:41:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T18:41:19,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:19,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:19,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091339279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091339283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091339284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091339284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,296 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091339289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091339390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091339399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091339399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091339399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,404 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:19,538 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108763850b2da554e7fb574927b980bf9f8_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108763850b2da554e7fb574927b980bf9f8_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/d8a1dfb8b5ca4b758ea7cef03a296a27, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/d8a1dfb8b5ca4b758ea7cef03a296a27 is 175, key is test_row_0/A:col10/1731091278130/Put/seqid=0 2024-11-08T18:41:19,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742163_1339 (size=30955) 2024-11-08T18:41:19,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091339600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,609 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T18:41:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091339609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091339609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,614 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091339609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,624 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091339909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091339919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,923 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091339920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,924 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:19,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091339929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:19,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:19,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:19,946 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/d8a1dfb8b5ca4b758ea7cef03a296a27 2024-11-08T18:41:19,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/38877608cdea41feb848c08fdd8e3b3b is 50, key is test_row_0/B:col10/1731091278130/Put/seqid=0 2024-11-08T18:41:19,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742164_1340 (size=12001) 2024-11-08T18:41:20,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T18:41:20,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:20,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091340309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:20,312 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 2d8b2888d7998b4dc696cf695019721f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:20,357 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/38877608cdea41feb848c08fdd8e3b3b 2024-11-08T18:41:20,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f20cdc1b249a441eaded75a2d0553224 is 50, key is test_row_0/C:col10/1731091278130/Put/seqid=0 2024-11-08T18:41:20,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742165_1341 (size=12001) 2024-11-08T18:41:20,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:20,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091340420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:20,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091340429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:20,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091340429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,437 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:20,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091340439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:20,445 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:20,767 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f20cdc1b249a441eaded75a2d0553224 2024-11-08T18:41:20,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/d8a1dfb8b5ca4b758ea7cef03a296a27 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/d8a1dfb8b5ca4b758ea7cef03a296a27 2024-11-08T18:41:20,774 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/d8a1dfb8b5ca4b758ea7cef03a296a27, entries=150, sequenceid=40, filesize=30.2 K 2024-11-08T18:41:20,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/38877608cdea41feb848c08fdd8e3b3b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/38877608cdea41feb848c08fdd8e3b3b 2024-11-08T18:41:20,778 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/38877608cdea41feb848c08fdd8e3b3b, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T18:41:20,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f20cdc1b249a441eaded75a2d0553224 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f20cdc1b249a441eaded75a2d0553224 2024-11-08T18:41:20,782 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f20cdc1b249a441eaded75a2d0553224, entries=150, sequenceid=40, filesize=11.7 K 2024-11-08T18:41:20,783 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2d8b2888d7998b4dc696cf695019721f in 1660ms, sequenceid=40, compaction requested=false 2024-11-08T18:41:20,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:20,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:20,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-08T18:41:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-08T18:41:20,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-08T18:41:20,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7330 sec 2024-11-08T18:41:20,789 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 2.7430 sec 2024-11-08T18:41:21,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:21,444 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:21,444 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T18:41:21,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108576e28ec8a404460b3ba65aa810805f2_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091281443/Put/seqid=0 2024-11-08T18:41:21,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742166_1342 (size=14594) 2024-11-08T18:41:21,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:21,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091341480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,493 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108576e28ec8a404460b3ba65aa810805f2_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108576e28ec8a404460b3ba65aa810805f2_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:21,494 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/2d49708a073a4e31b832387553645ed9, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:21,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/2d49708a073a4e31b832387553645ed9 is 175, key is test_row_0/A:col10/1731091281443/Put/seqid=0 2024-11-08T18:41:21,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742167_1343 (size=39549) 2024-11-08T18:41:21,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091341491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,498 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091341492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091341492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,502 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091341599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091341609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091341609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091341609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091341819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091341819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091341819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091341819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:21,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:21,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/2d49708a073a4e31b832387553645ed9 2024-11-08T18:41:21,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/9995d40298714ad19b4cfffdb5bd8bac is 50, key is test_row_0/B:col10/1731091281443/Put/seqid=0 2024-11-08T18:41:21,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742168_1344 (size=12001) 2024-11-08T18:41:22,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091342130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091342130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091342130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091342130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,136 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-08T18:41:22,219 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:22,219 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-08T18:41:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T18:41:22,221 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:22,222 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:22,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:22,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/9995d40298714ad19b4cfffdb5bd8bac 2024-11-08T18:41:22,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/8b509e24adc94ce7bc9c895454132a91 is 50, key is test_row_0/C:col10/1731091281443/Put/seqid=0 2024-11-08T18:41:22,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742169_1345 (size=12001) 2024-11-08T18:41:22,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T18:41:22,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091342339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,346 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 2d8b2888d7998b4dc696cf695019721f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T18:41:22,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:22,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T18:41:22,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:22,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T18:41:22,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091342640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,646 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091342649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091342649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091342649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:22,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:22,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T18:41:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:22,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/8b509e24adc94ce7bc9c895454132a91 2024-11-08T18:41:22,722 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/2d49708a073a4e31b832387553645ed9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2d49708a073a4e31b832387553645ed9 2024-11-08T18:41:22,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2d49708a073a4e31b832387553645ed9, entries=200, sequenceid=52, filesize=38.6 K 2024-11-08T18:41:22,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/9995d40298714ad19b4cfffdb5bd8bac as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/9995d40298714ad19b4cfffdb5bd8bac 2024-11-08T18:41:22,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/9995d40298714ad19b4cfffdb5bd8bac, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T18:41:22,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/8b509e24adc94ce7bc9c895454132a91 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8b509e24adc94ce7bc9c895454132a91 2024-11-08T18:41:22,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8b509e24adc94ce7bc9c895454132a91, entries=150, sequenceid=52, filesize=11.7 K 2024-11-08T18:41:22,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2d8b2888d7998b4dc696cf695019721f in 1290ms, sequenceid=52, compaction requested=true 2024-11-08T18:41:22,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:22,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:22,735 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:22,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:22,735 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:22,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:22,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:22,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:22,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:22,735 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:22,735 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:22,735 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/B is initiating minor compaction (all files) 2024-11-08T18:41:22,735 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/A is initiating minor compaction (all files) 2024-11-08T18:41:22,735 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/B in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,735 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/A in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,736 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/64048c569a1d442f9c94127f09b1dd46, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/d8a1dfb8b5ca4b758ea7cef03a296a27, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2d49708a073a4e31b832387553645ed9] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=99.1 K 2024-11-08T18:41:22,736 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/1745800b2d6e40b9b7436bdf77454a61, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/38877608cdea41feb848c08fdd8e3b3b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/9995d40298714ad19b4cfffdb5bd8bac] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=35.2 K 2024-11-08T18:41:22,736 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,736 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/64048c569a1d442f9c94127f09b1dd46, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/d8a1dfb8b5ca4b758ea7cef03a296a27, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2d49708a073a4e31b832387553645ed9] 2024-11-08T18:41:22,736 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1745800b2d6e40b9b7436bdf77454a61, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731091278052 2024-11-08T18:41:22,736 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64048c569a1d442f9c94127f09b1dd46, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731091278052 2024-11-08T18:41:22,736 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 38877608cdea41feb848c08fdd8e3b3b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731091278120 2024-11-08T18:41:22,737 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8a1dfb8b5ca4b758ea7cef03a296a27, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731091278120 2024-11-08T18:41:22,737 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9995d40298714ad19b4cfffdb5bd8bac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091279282 2024-11-08T18:41:22,737 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d49708a073a4e31b832387553645ed9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091279278 2024-11-08T18:41:22,741 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:22,743 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411083f89f7a9f9b44b998a7c3609e3573662_2d8b2888d7998b4dc696cf695019721f store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:22,743 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#B#compaction#295 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:22,743 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/8db0fc04e6d44b70876fc3baf9cb3f0d is 50, key is test_row_0/B:col10/1731091281443/Put/seqid=0 2024-11-08T18:41:22,744 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411083f89f7a9f9b44b998a7c3609e3573662_2d8b2888d7998b4dc696cf695019721f, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:22,744 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083f89f7a9f9b44b998a7c3609e3573662_2d8b2888d7998b4dc696cf695019721f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:22,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742170_1346 (size=4469) 2024-11-08T18:41:22,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742171_1347 (size=12104) 2024-11-08T18:41:22,764 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/8db0fc04e6d44b70876fc3baf9cb3f0d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/8db0fc04e6d44b70876fc3baf9cb3f0d 2024-11-08T18:41:22,768 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/B of 2d8b2888d7998b4dc696cf695019721f into 8db0fc04e6d44b70876fc3baf9cb3f0d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:22,768 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:22,769 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/B, priority=13, startTime=1731091282735; duration=0sec 2024-11-08T18:41:22,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:22,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:B 2024-11-08T18:41:22,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:22,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:22,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/C is initiating minor compaction (all files) 2024-11-08T18:41:22,770 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/C in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,770 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/7ebf9f9c3291475a80ac993d5b1ae7c5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f20cdc1b249a441eaded75a2d0553224, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8b509e24adc94ce7bc9c895454132a91] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=35.2 K 2024-11-08T18:41:22,770 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ebf9f9c3291475a80ac993d5b1ae7c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731091278052 2024-11-08T18:41:22,770 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f20cdc1b249a441eaded75a2d0553224, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731091278120 2024-11-08T18:41:22,770 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b509e24adc94ce7bc9c895454132a91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091279282 2024-11-08T18:41:22,776 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#C#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:22,776 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/2d7b239e914c4554ab8c5ee3143563ea is 50, key is test_row_0/C:col10/1731091281443/Put/seqid=0 2024-11-08T18:41:22,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742172_1348 (size=12104) 2024-11-08T18:41:22,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-08T18:41:22,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:22,832 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:41:22,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110800edfad603b14d19b8672357411c7e29_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091281466/Put/seqid=0 2024-11-08T18:41:22,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742173_1349 (size=12154) 2024-11-08T18:41:22,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T18:41:23,153 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#A#compaction#294 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:23,154 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/02a9644186ee48adb4d34dec6270ca5c is 175, key is test_row_0/A:col10/1731091281443/Put/seqid=0 2024-11-08T18:41:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742174_1350 (size=31058) 2024-11-08T18:41:23,187 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/2d7b239e914c4554ab8c5ee3143563ea as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2d7b239e914c4554ab8c5ee3143563ea 2024-11-08T18:41:23,192 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/C of 2d8b2888d7998b4dc696cf695019721f into 2d7b239e914c4554ab8c5ee3143563ea(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:23,192 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:23,192 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/C, priority=13, startTime=1731091282735; duration=0sec 2024-11-08T18:41:23,192 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:23,192 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:C 2024-11-08T18:41:23,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:23,245 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110800edfad603b14d19b8672357411c7e29_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110800edfad603b14d19b8672357411c7e29_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:23,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/b01f2ae840364f6db2d04cd3b0374bca, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:23,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/b01f2ae840364f6db2d04cd3b0374bca is 175, key is test_row_0/A:col10/1731091281466/Put/seqid=0 2024-11-08T18:41:23,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742175_1351 (size=30955) 2024-11-08T18:41:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T18:41:23,562 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/02a9644186ee48adb4d34dec6270ca5c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/02a9644186ee48adb4d34dec6270ca5c 2024-11-08T18:41:23,565 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/A of 2d8b2888d7998b4dc696cf695019721f into 02a9644186ee48adb4d34dec6270ca5c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:23,566 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:23,566 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/A, priority=13, startTime=1731091282734; duration=0sec 2024-11-08T18:41:23,566 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:23,566 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:A 2024-11-08T18:41:23,650 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/b01f2ae840364f6db2d04cd3b0374bca 2024-11-08T18:41:23,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/73e7e326ede94756a49eed9b2460dde6 is 50, key is test_row_0/B:col10/1731091281466/Put/seqid=0 2024-11-08T18:41:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742176_1352 (size=12001) 2024-11-08T18:41:23,663 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/73e7e326ede94756a49eed9b2460dde6 2024-11-08T18:41:23,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:23,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:23,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f06a6b6080724816a5650baac0451843 is 50, key is test_row_0/C:col10/1731091281466/Put/seqid=0 2024-11-08T18:41:23,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742177_1353 (size=12001) 2024-11-08T18:41:23,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091343684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:23,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:23,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091343685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:23,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:23,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091343686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,695 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:23,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091343687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:23,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091343789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:23,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:23,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091343789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:23,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091343799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:23,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:23,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091343810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:23,811 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091344009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091344009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091344009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,013 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091344019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,027 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,081 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f06a6b6080724816a5650baac0451843 2024-11-08T18:41:24,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/b01f2ae840364f6db2d04cd3b0374bca as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/b01f2ae840364f6db2d04cd3b0374bca 2024-11-08T18:41:24,087 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/b01f2ae840364f6db2d04cd3b0374bca, entries=150, sequenceid=77, filesize=30.2 K 2024-11-08T18:41:24,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/73e7e326ede94756a49eed9b2460dde6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/73e7e326ede94756a49eed9b2460dde6 2024-11-08T18:41:24,091 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/73e7e326ede94756a49eed9b2460dde6, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T18:41:24,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f06a6b6080724816a5650baac0451843 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f06a6b6080724816a5650baac0451843 2024-11-08T18:41:24,094 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f06a6b6080724816a5650baac0451843, entries=150, sequenceid=77, filesize=11.7 K 2024-11-08T18:41:24,095 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 2d8b2888d7998b4dc696cf695019721f in 1263ms, sequenceid=77, compaction requested=false 2024-11-08T18:41:24,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:24,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-08T18:41:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-08T18:41:24,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-08T18:41:24,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8740 sec 2024-11-08T18:41:24,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.8800 sec 2024-11-08T18:41:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:24,324 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:24,324 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:41:24,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080bf8446e08e94102aaa003c5179b127c_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091283684/Put/seqid=0 2024-11-08T18:41:24,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742178_1354 (size=17034) 2024-11-08T18:41:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-08T18:41:24,370 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:24,370 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-08T18:41:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T18:41:24,371 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:24,372 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:24,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091344361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091344362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,376 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091344368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091344369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,379 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T18:41:24,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091344479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091344479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091344480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,486 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091344489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T18:41:24,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:24,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T18:41:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T18:41:24,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091344689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091344690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091344690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,704 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091344699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:24,704 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:24,747 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:24,750 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080bf8446e08e94102aaa003c5179b127c_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080bf8446e08e94102aaa003c5179b127c_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:24,751 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/f6bc249162e84a5caaa1a19454740d67, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:24,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/f6bc249162e84a5caaa1a19454740d67 is 175, key is test_row_0/A:col10/1731091283684/Put/seqid=0 2024-11-08T18:41:24,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742179_1355 (size=48139) 2024-11-08T18:41:24,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T18:41:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T18:41:24,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:24,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:24,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T18:41:25,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091344999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:25,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091344999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:25,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091344999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:25,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091345010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:25,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T18:41:25,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:25,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,157 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=94, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/f6bc249162e84a5caaa1a19454740d67 2024-11-08T18:41:25,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/f64e0806e0cc46b78a0721ec4932b922 is 50, key is test_row_0/B:col10/1731091283684/Put/seqid=0 2024-11-08T18:41:25,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742180_1356 (size=12001) 2024-11-08T18:41:25,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T18:41:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:25,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T18:41:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T18:41:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091345519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091345519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091345520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:25,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:25,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091345520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:25,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:25,569 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/f64e0806e0cc46b78a0721ec4932b922 2024-11-08T18:41:25,575 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/cebc20b892a64a0381b56ee2e9beb47b is 50, key is test_row_0/C:col10/1731091283684/Put/seqid=0 2024-11-08T18:41:25,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742181_1357 (size=12001) 2024-11-08T18:41:25,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=94 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/cebc20b892a64a0381b56ee2e9beb47b 2024-11-08T18:41:25,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/f6bc249162e84a5caaa1a19454740d67 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f6bc249162e84a5caaa1a19454740d67 2024-11-08T18:41:25,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f6bc249162e84a5caaa1a19454740d67, entries=250, sequenceid=94, filesize=47.0 K 2024-11-08T18:41:25,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/f64e0806e0cc46b78a0721ec4932b922 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f64e0806e0cc46b78a0721ec4932b922 2024-11-08T18:41:25,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f64e0806e0cc46b78a0721ec4932b922, entries=150, sequenceid=94, filesize=11.7 K 2024-11-08T18:41:25,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/cebc20b892a64a0381b56ee2e9beb47b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/cebc20b892a64a0381b56ee2e9beb47b 2024-11-08T18:41:25,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/cebc20b892a64a0381b56ee2e9beb47b, entries=150, sequenceid=94, filesize=11.7 K 2024-11-08T18:41:25,593 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 2d8b2888d7998b4dc696cf695019721f in 1269ms, sequenceid=94, compaction requested=true 2024-11-08T18:41:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:25,593 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:25,593 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:25,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:25,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-08T18:41:25,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,594 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T18:41:25,596 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:25,596 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110152 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:25,596 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/A is initiating minor compaction (all files) 2024-11-08T18:41:25,596 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/B is initiating minor compaction (all files) 2024-11-08T18:41:25,596 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/A in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,596 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/B in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,596 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/02a9644186ee48adb4d34dec6270ca5c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/b01f2ae840364f6db2d04cd3b0374bca, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f6bc249162e84a5caaa1a19454740d67] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=107.6 K 2024-11-08T18:41:25,596 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/8db0fc04e6d44b70876fc3baf9cb3f0d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/73e7e326ede94756a49eed9b2460dde6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f64e0806e0cc46b78a0721ec4932b922] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=35.3 K 2024-11-08T18:41:25,596 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:25,596 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/02a9644186ee48adb4d34dec6270ca5c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/b01f2ae840364f6db2d04cd3b0374bca, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f6bc249162e84a5caaa1a19454740d67] 2024-11-08T18:41:25,597 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8db0fc04e6d44b70876fc3baf9cb3f0d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091279282 2024-11-08T18:41:25,597 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 02a9644186ee48adb4d34dec6270ca5c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091279282 2024-11-08T18:41:25,597 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 73e7e326ede94756a49eed9b2460dde6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091281466 2024-11-08T18:41:25,597 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b01f2ae840364f6db2d04cd3b0374bca, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091281466 2024-11-08T18:41:25,597 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f64e0806e0cc46b78a0721ec4932b922, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091283684 2024-11-08T18:41:25,597 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6bc249162e84a5caaa1a19454740d67, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091283676 2024-11-08T18:41:25,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083bf83033b8654e67aeacda12e4ba6628_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091284368/Put/seqid=0 2024-11-08T18:41:25,614 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:25,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742182_1358 (size=12154) 2024-11-08T18:41:25,616 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#B#compaction#305 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:25,617 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411087d73842c8aed4bf4833ec259adfd1aed_2d8b2888d7998b4dc696cf695019721f store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:25,617 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/acf6b83414f84f9daf929241b3b570f2 is 50, key is test_row_0/B:col10/1731091283684/Put/seqid=0 2024-11-08T18:41:25,619 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411087d73842c8aed4bf4833ec259adfd1aed_2d8b2888d7998b4dc696cf695019721f, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:25,619 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411087d73842c8aed4bf4833ec259adfd1aed_2d8b2888d7998b4dc696cf695019721f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:25,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742183_1359 (size=12207) 2024-11-08T18:41:25,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742184_1360 (size=4469) 2024-11-08T18:41:25,632 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#A#compaction#304 average throughput is 1.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:25,632 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/fa3b9c779066429e9327b12419b2bc7a is 175, key is test_row_0/A:col10/1731091283684/Put/seqid=0 2024-11-08T18:41:25,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742185_1361 (size=31161) 2024-11-08T18:41:26,015 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,018 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083bf83033b8654e67aeacda12e4ba6628_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083bf83033b8654e67aeacda12e4ba6628_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:26,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/7a0b0cf0079445029c0c712d110ad72d, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:26,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/7a0b0cf0079445029c0c712d110ad72d is 175, key is test_row_0/A:col10/1731091284368/Put/seqid=0 2024-11-08T18:41:26,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742186_1362 (size=30955) 2024-11-08T18:41:26,024 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/7a0b0cf0079445029c0c712d110ad72d 2024-11-08T18:41:26,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/28ee43d436a54b1c807aaf210bdd6be3 is 50, key is test_row_0/B:col10/1731091284368/Put/seqid=0 2024-11-08T18:41:26,031 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/acf6b83414f84f9daf929241b3b570f2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/acf6b83414f84f9daf929241b3b570f2 2024-11-08T18:41:26,035 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/B of 2d8b2888d7998b4dc696cf695019721f into acf6b83414f84f9daf929241b3b570f2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:26,035 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:26,035 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/B, priority=13, startTime=1731091285593; duration=0sec 2024-11-08T18:41:26,035 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:26,035 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:B 2024-11-08T18:41:26,035 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:26,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742187_1363 (size=12001) 2024-11-08T18:41:26,037 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:26,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/28ee43d436a54b1c807aaf210bdd6be3 2024-11-08T18:41:26,037 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/C is initiating minor compaction (all files) 2024-11-08T18:41:26,038 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/C in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:26,038 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2d7b239e914c4554ab8c5ee3143563ea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f06a6b6080724816a5650baac0451843, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/cebc20b892a64a0381b56ee2e9beb47b] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=35.3 K 2024-11-08T18:41:26,038 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d7b239e914c4554ab8c5ee3143563ea, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731091279282 2024-11-08T18:41:26,038 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f06a6b6080724816a5650baac0451843, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731091281466 2024-11-08T18:41:26,038 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting cebc20b892a64a0381b56ee2e9beb47b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091283684 2024-11-08T18:41:26,045 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/fa3b9c779066429e9327b12419b2bc7a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/fa3b9c779066429e9327b12419b2bc7a 2024-11-08T18:41:26,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/2816bf7cbccd438cb6be681c5d8f3c14 is 50, key is test_row_0/C:col10/1731091284368/Put/seqid=0 2024-11-08T18:41:26,046 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#C#compaction#308 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:26,046 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/9b34d28b916941b8b9af7f762a1f1d00 is 50, key is test_row_0/C:col10/1731091283684/Put/seqid=0 2024-11-08T18:41:26,050 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/A of 2d8b2888d7998b4dc696cf695019721f into fa3b9c779066429e9327b12419b2bc7a(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:26,050 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:26,050 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/A, priority=13, startTime=1731091285593; duration=0sec 2024-11-08T18:41:26,050 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:26,050 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:A 2024-11-08T18:41:26,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742189_1365 (size=12207) 2024-11-08T18:41:26,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742188_1364 (size=12001) 2024-11-08T18:41:26,057 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/2816bf7cbccd438cb6be681c5d8f3c14 2024-11-08T18:41:26,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/7a0b0cf0079445029c0c712d110ad72d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/7a0b0cf0079445029c0c712d110ad72d 2024-11-08T18:41:26,066 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/7a0b0cf0079445029c0c712d110ad72d, entries=150, sequenceid=115, filesize=30.2 K 2024-11-08T18:41:26,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/28ee43d436a54b1c807aaf210bdd6be3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/28ee43d436a54b1c807aaf210bdd6be3 2024-11-08T18:41:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,070 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/28ee43d436a54b1c807aaf210bdd6be3, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T18:41:26,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/2816bf7cbccd438cb6be681c5d8f3c14 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2816bf7cbccd438cb6be681c5d8f3c14 2024-11-08T18:41:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,074 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2816bf7cbccd438cb6be681c5d8f3c14, entries=150, sequenceid=115, filesize=11.7 K 2024-11-08T18:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,075 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=0 B/0 for 2d8b2888d7998b4dc696cf695019721f in 481ms, sequenceid=115, compaction requested=false 2024-11-08T18:41:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:26,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-08T18:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-08T18:41:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-08T18:41:26,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7040 sec 2024-11-08T18:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.7090 sec 2024-11-08T18:41:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,491 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/9b34d28b916941b8b9af7f762a1f1d00 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/9b34d28b916941b8b9af7f762a1f1d00 2024-11-08T18:41:26,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,495 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/C of 2d8b2888d7998b4dc696cf695019721f into 9b34d28b916941b8b9af7f762a1f1d00(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:26,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,495 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:26,495 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/C, priority=13, startTime=1731091285593; duration=0sec 2024-11-08T18:41:26,495 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:26,495 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:C 2024-11-08T18:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-08T18:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,520 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,520 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-08T18:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T18:41:26,522 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,523 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:26,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:26,563 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:26,563 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:41:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b433dddb1647475394ccb0166645d6a3_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091286562/Put/seqid=0 2024-11-08T18:41:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742190_1366 (size=21968) 2024-11-08T18:41:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091346612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,626 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091346618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,626 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091346619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,628 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T18:41:26,635 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091346626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,636 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091346626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,636 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:26,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091346729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091346730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091346739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091346739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091346739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:26,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:26,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T18:41:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091346939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091346949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,953 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091346949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091346949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,954 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,965 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:26,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091346959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:26,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:26,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:26,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:26,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:26,985 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:26,988 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b433dddb1647475394ccb0166645d6a3_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b433dddb1647475394ccb0166645d6a3_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:26,989 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/3cd8f8a872c741a2a2bc44245d31ad61, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:26,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/3cd8f8a872c741a2a2bc44245d31ad61 is 175, key is test_row_0/A:col10/1731091286562/Put/seqid=0 2024-11-08T18:41:26,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742191_1367 (size=65373) 2024-11-08T18:41:27,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T18:41:27,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091347250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091347259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091347259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091347259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091347269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,394 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/3cd8f8a872c741a2a2bc44245d31ad61 2024-11-08T18:41:27,404 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/f0966c276510487fa0ecd783ec9bc298 is 50, key is test_row_0/B:col10/1731091286562/Put/seqid=0 2024-11-08T18:41:27,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742192_1368 (size=12051) 2024-11-08T18:41:27,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/f0966c276510487fa0ecd783ec9bc298 2024-11-08T18:41:27,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/098e4fd03a8e44608723cb9ff8136d7d is 50, key is test_row_0/C:col10/1731091286562/Put/seqid=0 2024-11-08T18:41:27,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742193_1369 (size=12051) 2024-11-08T18:41:27,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,592 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T18:41:27,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:27,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091347760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,775 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091347769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091347769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091347769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,776 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:27,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091347789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:27,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:27,820 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/098e4fd03a8e44608723cb9ff8136d7d 2024-11-08T18:41:27,824 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/3cd8f8a872c741a2a2bc44245d31ad61 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/3cd8f8a872c741a2a2bc44245d31ad61 2024-11-08T18:41:27,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/3cd8f8a872c741a2a2bc44245d31ad61, entries=350, sequenceid=129, filesize=63.8 K 2024-11-08T18:41:27,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/f0966c276510487fa0ecd783ec9bc298 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f0966c276510487fa0ecd783ec9bc298 2024-11-08T18:41:27,831 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f0966c276510487fa0ecd783ec9bc298, entries=150, sequenceid=129, filesize=11.8 K 2024-11-08T18:41:27,831 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/098e4fd03a8e44608723cb9ff8136d7d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/098e4fd03a8e44608723cb9ff8136d7d 2024-11-08T18:41:27,834 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/098e4fd03a8e44608723cb9ff8136d7d, entries=150, sequenceid=129, filesize=11.8 K 2024-11-08T18:41:27,835 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2d8b2888d7998b4dc696cf695019721f in 1272ms, sequenceid=129, compaction requested=true 2024-11-08T18:41:27,835 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:27,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:27,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:27,835 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:27,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:27,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:27,835 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:27,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:27,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:27,836 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127489 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:27,836 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/A is initiating minor compaction (all files) 2024-11-08T18:41:27,836 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/A in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,836 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/fa3b9c779066429e9327b12419b2bc7a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/7a0b0cf0079445029c0c712d110ad72d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/3cd8f8a872c741a2a2bc44245d31ad61] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=124.5 K 2024-11-08T18:41:27,836 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,837 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/fa3b9c779066429e9327b12419b2bc7a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/7a0b0cf0079445029c0c712d110ad72d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/3cd8f8a872c741a2a2bc44245d31ad61] 2024-11-08T18:41:27,837 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa3b9c779066429e9327b12419b2bc7a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091283684 2024-11-08T18:41:27,837 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:27,837 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a0b0cf0079445029c0c712d110ad72d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731091284347 2024-11-08T18:41:27,837 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/B is initiating minor compaction (all files) 2024-11-08T18:41:27,837 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/B in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,837 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/acf6b83414f84f9daf929241b3b570f2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/28ee43d436a54b1c807aaf210bdd6be3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f0966c276510487fa0ecd783ec9bc298] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=35.4 K 2024-11-08T18:41:27,837 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cd8f8a872c741a2a2bc44245d31ad61, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731091286444 2024-11-08T18:41:27,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting acf6b83414f84f9daf929241b3b570f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091283684 2024-11-08T18:41:27,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ee43d436a54b1c807aaf210bdd6be3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731091284347 2024-11-08T18:41:27,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f0966c276510487fa0ecd783ec9bc298, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731091286444 2024-11-08T18:41:27,843 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:27,845 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411083c57037545cd46c881c272294509b69f_2d8b2888d7998b4dc696cf695019721f store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:27,845 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#B#compaction#313 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:27,845 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/06546d247b3f4e7b819221ec66b7c59e is 50, key is test_row_0/B:col10/1731091286562/Put/seqid=0 2024-11-08T18:41:27,847 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411083c57037545cd46c881c272294509b69f_2d8b2888d7998b4dc696cf695019721f, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:27,847 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083c57037545cd46c881c272294509b69f_2d8b2888d7998b4dc696cf695019721f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:27,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742194_1370 (size=12359) 2024-11-08T18:41:27,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742195_1371 (size=4469) 2024-11-08T18:41:27,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-08T18:41:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:27,900 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T18:41:27,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c2df614dd66b4a3c82e891809c1ec295_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091286624/Put/seqid=0 2024-11-08T18:41:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742196_1372 (size=12304) 2024-11-08T18:41:27,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:27,915 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c2df614dd66b4a3c82e891809c1ec295_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c2df614dd66b4a3c82e891809c1ec295_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:27,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/0e47800e7a2a49dfaa3ec0e70498e8ea, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:27,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/0e47800e7a2a49dfaa3ec0e70498e8ea is 175, key is test_row_0/A:col10/1731091286624/Put/seqid=0 2024-11-08T18:41:27,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742197_1373 (size=31105) 2024-11-08T18:41:28,253 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#A#compaction#312 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:28,253 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/6a3af0f13fb544cdbd683a978bd7056f is 175, key is test_row_0/A:col10/1731091286562/Put/seqid=0 2024-11-08T18:41:28,254 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/06546d247b3f4e7b819221ec66b7c59e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/06546d247b3f4e7b819221ec66b7c59e 2024-11-08T18:41:28,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742198_1374 (size=31313) 2024-11-08T18:41:28,258 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/B of 2d8b2888d7998b4dc696cf695019721f into 06546d247b3f4e7b819221ec66b7c59e(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:28,258 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:28,258 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/B, priority=13, startTime=1731091287835; duration=0sec 2024-11-08T18:41:28,258 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:28,258 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:B 2024-11-08T18:41:28,258 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:28,259 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:28,259 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/C is initiating minor compaction (all files) 2024-11-08T18:41:28,259 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/C in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:28,259 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/9b34d28b916941b8b9af7f762a1f1d00, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2816bf7cbccd438cb6be681c5d8f3c14, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/098e4fd03a8e44608723cb9ff8136d7d] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=35.4 K 2024-11-08T18:41:28,260 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b34d28b916941b8b9af7f762a1f1d00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=94, earliestPutTs=1731091283684 2024-11-08T18:41:28,260 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2816bf7cbccd438cb6be681c5d8f3c14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731091284347 2024-11-08T18:41:28,260 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 098e4fd03a8e44608723cb9ff8136d7d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731091286444 2024-11-08T18:41:28,265 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#C#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:28,266 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/0360b5bf69ff49e984d9f11c84c8cf65 is 50, key is test_row_0/C:col10/1731091286562/Put/seqid=0 2024-11-08T18:41:28,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742199_1375 (size=12359) 2024-11-08T18:41:28,272 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/0360b5bf69ff49e984d9f11c84c8cf65 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0360b5bf69ff49e984d9f11c84c8cf65 2024-11-08T18:41:28,276 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/C of 2d8b2888d7998b4dc696cf695019721f into 0360b5bf69ff49e984d9f11c84c8cf65(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:28,276 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:28,276 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/C, priority=13, startTime=1731091287835; duration=0sec 2024-11-08T18:41:28,276 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:28,276 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:C 2024-11-08T18:41:28,320 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/0e47800e7a2a49dfaa3ec0e70498e8ea 2024-11-08T18:41:28,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/b1d96ad5012442e29de664515882dcde is 50, key is test_row_0/B:col10/1731091286624/Put/seqid=0 2024-11-08T18:41:28,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742200_1376 (size=12151) 2024-11-08T18:41:28,336 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/b1d96ad5012442e29de664515882dcde 2024-11-08T18:41:28,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/8278af79100c40dbbeed5b20a114aee2 is 50, key is test_row_0/C:col10/1731091286624/Put/seqid=0 2024-11-08T18:41:28,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742201_1377 (size=12151) 2024-11-08T18:41:28,661 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/6a3af0f13fb544cdbd683a978bd7056f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/6a3af0f13fb544cdbd683a978bd7056f 2024-11-08T18:41:28,664 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/A of 2d8b2888d7998b4dc696cf695019721f into 6a3af0f13fb544cdbd683a978bd7056f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:28,664 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:28,664 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/A, priority=13, startTime=1731091287835; duration=0sec 2024-11-08T18:41:28,664 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:28,664 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:A 2024-11-08T18:41:28,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T18:41:28,749 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/8278af79100c40dbbeed5b20a114aee2 2024-11-08T18:41:28,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/0e47800e7a2a49dfaa3ec0e70498e8ea as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0e47800e7a2a49dfaa3ec0e70498e8ea 2024-11-08T18:41:28,756 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0e47800e7a2a49dfaa3ec0e70498e8ea, entries=150, sequenceid=154, filesize=30.4 K 2024-11-08T18:41:28,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/b1d96ad5012442e29de664515882dcde as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/b1d96ad5012442e29de664515882dcde 2024-11-08T18:41:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,760 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/b1d96ad5012442e29de664515882dcde, entries=150, sequenceid=154, filesize=11.9 K 2024-11-08T18:41:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/8278af79100c40dbbeed5b20a114aee2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8278af79100c40dbbeed5b20a114aee2 2024-11-08T18:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,764 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8278af79100c40dbbeed5b20a114aee2, entries=150, sequenceid=154, filesize=11.9 K 2024-11-08T18:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,765 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 2d8b2888d7998b4dc696cf695019721f in 865ms, sequenceid=154, compaction requested=false 2024-11-08T18:41:28,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:28,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-08T18:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-08T18:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-08T18:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,768 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2430 sec 2024-11-08T18:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,769 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.2480 sec 2024-11-08T18:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,890 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:28,890 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:41:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110853f7d1fff2594fcf89820f386ce06714_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091288889/Put/seqid=0 2024-11-08T18:41:28,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742202_1378 (size=22268) 2024-11-08T18:41:28,925 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:28,931 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110853f7d1fff2594fcf89820f386ce06714_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110853f7d1fff2594fcf89820f386ce06714_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:28,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/233694830ca242f3877ecb9a5c422b69, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:28,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/233694830ca242f3877ecb9a5c422b69 is 175, key is test_row_0/A:col10/1731091288889/Put/seqid=0 2024-11-08T18:41:28,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742203_1379 (size=65673) 2024-11-08T18:41:28,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:28,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091348945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:28,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091348946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:28,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091348946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:28,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091348947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,957 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:28,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:28,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091348950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:28,960 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091349059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091349059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091349059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091349059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,066 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,067 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091349069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,077 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091349269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,271 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091349269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091349269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,272 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091349269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091349279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,345 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/233694830ca242f3877ecb9a5c422b69 2024-11-08T18:41:29,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/c374421a10734c478e59839200814a39 is 50, key is test_row_0/B:col10/1731091288889/Put/seqid=0 2024-11-08T18:41:29,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742204_1380 (size=12151) 2024-11-08T18:41:29,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091349580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091349580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091349580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,587 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091349580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,588 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:29,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091349589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:29,596 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:29,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/c374421a10734c478e59839200814a39 2024-11-08T18:41:29,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/3419aef58c1e43e38b9b8f48bf198373 is 50, key is test_row_0/C:col10/1731091288889/Put/seqid=0 2024-11-08T18:41:29,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742205_1381 (size=12151) 2024-11-08T18:41:30,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:30,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091350089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,093 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091350099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:30,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091350099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:30,106 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:30,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091350099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,107 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:30,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:30,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091350109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:30,116 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:30,166 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/3419aef58c1e43e38b9b8f48bf198373 2024-11-08T18:41:30,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/233694830ca242f3877ecb9a5c422b69 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/233694830ca242f3877ecb9a5c422b69 2024-11-08T18:41:30,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/233694830ca242f3877ecb9a5c422b69, entries=350, sequenceid=168, filesize=64.1 K 2024-11-08T18:41:30,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/c374421a10734c478e59839200814a39 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/c374421a10734c478e59839200814a39 2024-11-08T18:41:30,176 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/c374421a10734c478e59839200814a39, entries=150, sequenceid=168, filesize=11.9 K 2024-11-08T18:41:30,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/3419aef58c1e43e38b9b8f48bf198373 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/3419aef58c1e43e38b9b8f48bf198373 2024-11-08T18:41:30,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/3419aef58c1e43e38b9b8f48bf198373, entries=150, sequenceid=168, filesize=11.9 K 2024-11-08T18:41:30,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 2d8b2888d7998b4dc696cf695019721f in 1290ms, sequenceid=168, compaction requested=true 2024-11-08T18:41:30,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:30,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:30,181 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:30,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:30,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:30,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:30,181 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:30,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:30,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:30,182 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:30,182 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/B is initiating minor compaction (all files) 2024-11-08T18:41:30,182 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/B in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:30,182 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/06546d247b3f4e7b819221ec66b7c59e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/b1d96ad5012442e29de664515882dcde, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/c374421a10734c478e59839200814a39] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=35.8 K 2024-11-08T18:41:30,182 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128091 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:30,182 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/A is initiating minor compaction (all files) 2024-11-08T18:41:30,182 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/A in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:30,182 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/6a3af0f13fb544cdbd683a978bd7056f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0e47800e7a2a49dfaa3ec0e70498e8ea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/233694830ca242f3877ecb9a5c422b69] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=125.1 K 2024-11-08T18:41:30,182 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:30,182 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/6a3af0f13fb544cdbd683a978bd7056f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0e47800e7a2a49dfaa3ec0e70498e8ea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/233694830ca242f3877ecb9a5c422b69] 2024-11-08T18:41:30,183 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 06546d247b3f4e7b819221ec66b7c59e, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731091286444 2024-11-08T18:41:30,183 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a3af0f13fb544cdbd683a978bd7056f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731091286444 2024-11-08T18:41:30,183 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting b1d96ad5012442e29de664515882dcde, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731091286615 2024-11-08T18:41:30,183 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e47800e7a2a49dfaa3ec0e70498e8ea, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731091286615 2024-11-08T18:41:30,183 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting c374421a10734c478e59839200814a39, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731091288860 2024-11-08T18:41:30,183 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 233694830ca242f3877ecb9a5c422b69, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731091288834 2024-11-08T18:41:30,194 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:30,196 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#B#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:30,197 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/0bb7ef03b3e549e59bdc32318da77237 is 50, key is test_row_0/B:col10/1731091288889/Put/seqid=0 2024-11-08T18:41:30,199 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108d265faa5afa64c9eb43e63c408c38d9d_2d8b2888d7998b4dc696cf695019721f store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:30,201 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108d265faa5afa64c9eb43e63c408c38d9d_2d8b2888d7998b4dc696cf695019721f, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:30,201 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d265faa5afa64c9eb43e63c408c38d9d_2d8b2888d7998b4dc696cf695019721f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:30,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742206_1382 (size=12561) 2024-11-08T18:41:30,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742207_1383 (size=4469) 2024-11-08T18:41:30,210 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/0bb7ef03b3e549e59bdc32318da77237 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0bb7ef03b3e549e59bdc32318da77237 2024-11-08T18:41:30,218 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/B of 2d8b2888d7998b4dc696cf695019721f into 0bb7ef03b3e549e59bdc32318da77237(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:30,218 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:30,219 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/B, priority=13, startTime=1731091290181; duration=0sec 2024-11-08T18:41:30,219 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:30,219 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:B 2024-11-08T18:41:30,219 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:30,220 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:30,220 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/C is initiating minor compaction (all files) 2024-11-08T18:41:30,220 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/C in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:30,221 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0360b5bf69ff49e984d9f11c84c8cf65, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8278af79100c40dbbeed5b20a114aee2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/3419aef58c1e43e38b9b8f48bf198373] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=35.8 K 2024-11-08T18:41:30,221 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0360b5bf69ff49e984d9f11c84c8cf65, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731091286444 2024-11-08T18:41:30,221 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8278af79100c40dbbeed5b20a114aee2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731091286615 2024-11-08T18:41:30,221 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3419aef58c1e43e38b9b8f48bf198373, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731091288860 2024-11-08T18:41:30,228 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:30,228 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/1679da2bd1fb42fb903f35ffa4abbfc3 is 50, key is test_row_0/C:col10/1731091288889/Put/seqid=0 2024-11-08T18:41:30,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742208_1384 (size=12561) 2024-11-08T18:41:30,608 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#A#compaction#321 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:30,608 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/109cfcb8d50a4bb4b9ec1f0c8bdc80bf is 175, key is test_row_0/A:col10/1731091288889/Put/seqid=0 2024-11-08T18:41:30,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742209_1385 (size=31515) 2024-11-08T18:41:30,635 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/1679da2bd1fb42fb903f35ffa4abbfc3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/1679da2bd1fb42fb903f35ffa4abbfc3 2024-11-08T18:41:30,639 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/C of 2d8b2888d7998b4dc696cf695019721f into 1679da2bd1fb42fb903f35ffa4abbfc3(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:30,639 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:30,639 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/C, priority=13, startTime=1731091290181; duration=0sec 2024-11-08T18:41:30,639 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:30,639 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:C 2024-11-08T18:41:30,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-08T18:41:30,689 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:30,689 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:30,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-08T18:41:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T18:41:30,691 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:30,691 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:30,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:30,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T18:41:30,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-08T18:41:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:30,843 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T18:41:30,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085390edf9b88646858500d4fa600c4d34_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091288945/Put/seqid=0 2024-11-08T18:41:30,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742210_1386 (size=12304) 2024-11-08T18:41:30,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:30,859 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085390edf9b88646858500d4fa600c4d34_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085390edf9b88646858500d4fa600c4d34_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:30,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/28ab82aabe3549498e77cf241c34c052, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:30,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/28ab82aabe3549498e77cf241c34c052 is 175, key is test_row_0/A:col10/1731091288945/Put/seqid=0 2024-11-08T18:41:30,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742211_1387 (size=31105) 2024-11-08T18:41:31,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T18:41:31,018 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/109cfcb8d50a4bb4b9ec1f0c8bdc80bf as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/109cfcb8d50a4bb4b9ec1f0c8bdc80bf 2024-11-08T18:41:31,022 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/A of 2d8b2888d7998b4dc696cf695019721f into 109cfcb8d50a4bb4b9ec1f0c8bdc80bf(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:31,022 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:31,022 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/A, priority=13, startTime=1731091290181; duration=0sec 2024-11-08T18:41:31,022 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:31,022 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:A 2024-11-08T18:41:31,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:31,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:31,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091351122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091351123, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091351124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091351124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091351129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,132 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2d8b2888d7998b4dc696cf695019721f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091351230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091351239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091351239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091351239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,266 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/28ab82aabe3549498e77cf241c34c052 2024-11-08T18:41:31,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/5a0b5ae1140b4a398fa111c2f23052ab is 50, key is test_row_0/B:col10/1731091288945/Put/seqid=0 2024-11-08T18:41:31,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742212_1388 (size=12151) 2024-11-08T18:41:31,277 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/5a0b5ae1140b4a398fa111c2f23052ab 2024-11-08T18:41:31,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/2b33ce6684144a359c6aebb45bda5095 is 50, key is test_row_0/C:col10/1731091288945/Put/seqid=0 2024-11-08T18:41:31,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742213_1389 (size=12151) 2024-11-08T18:41:31,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T18:41:31,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091351439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091351449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,453 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091351449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,454 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091351449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,455 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,456 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,690 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/2b33ce6684144a359c6aebb45bda5095 2024-11-08T18:41:31,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/28ab82aabe3549498e77cf241c34c052 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/28ab82aabe3549498e77cf241c34c052 2024-11-08T18:41:31,696 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/28ab82aabe3549498e77cf241c34c052, entries=150, sequenceid=195, filesize=30.4 K 2024-11-08T18:41:31,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/5a0b5ae1140b4a398fa111c2f23052ab as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5a0b5ae1140b4a398fa111c2f23052ab 2024-11-08T18:41:31,700 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5a0b5ae1140b4a398fa111c2f23052ab, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T18:41:31,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/2b33ce6684144a359c6aebb45bda5095 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2b33ce6684144a359c6aebb45bda5095 2024-11-08T18:41:31,703 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2b33ce6684144a359c6aebb45bda5095, entries=150, sequenceid=195, filesize=11.9 K 2024-11-08T18:41:31,704 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 2d8b2888d7998b4dc696cf695019721f in 861ms, sequenceid=195, compaction requested=false 2024-11-08T18:41:31,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:31,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:31,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-08T18:41:31,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-08T18:41:31,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-08T18:41:31,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0140 sec 2024-11-08T18:41:31,711 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.0210 sec 2024-11-08T18:41:31,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:31,755 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:31,755 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T18:41:31,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080c5b3f20948a4857ae9a94834caca7ac_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091291123/Put/seqid=0 2024-11-08T18:41:31,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742214_1390 (size=14794) 2024-11-08T18:41:31,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091351801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091351801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091351803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091351804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-08T18:41:31,830 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:31,830 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-08T18:41:31,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T18:41:31,832 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:31,832 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:31,832 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:31,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091351909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091351920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091351920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,924 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091351920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:31,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:31,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T18:41:31,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:31,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:31,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:31,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:31,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:31,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:31,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091352119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,125 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091352130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091352130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,134 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091352130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:32,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:32,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T18:41:32,171 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:32,174 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080c5b3f20948a4857ae9a94834caca7ac_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c5b3f20948a4857ae9a94834caca7ac_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:32,175 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/996c15fbecb548aa8261bd6dd1a61282, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:32,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/996c15fbecb548aa8261bd6dd1a61282 is 175, key is test_row_0/A:col10/1731091291123/Put/seqid=0 2024-11-08T18:41:32,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742215_1391 (size=39749) 2024-11-08T18:41:32,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:32,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:32,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091352429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,435 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:32,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:32,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,442 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091352439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091352439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091352439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,444 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T18:41:32,579 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/996c15fbecb548aa8261bd6dd1a61282 2024-11-08T18:41:32,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/d061740139834ba5be1eeecba47312f1 is 50, key is test_row_0/B:col10/1731091291123/Put/seqid=0 2024-11-08T18:41:32,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742216_1392 (size=12151) 2024-11-08T18:41:32,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:32,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,595 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:32,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:32,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:32,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:32,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:32,900 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:32,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091352939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091352949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,956 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,956 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091352950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091352950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:32,957 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:32,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T18:41:32,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/d061740139834ba5be1eeecba47312f1 2024-11-08T18:41:32,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/6e12d5b0113941eab291093be48651c6 is 50, key is test_row_0/C:col10/1731091291123/Put/seqid=0 2024-11-08T18:41:32,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742217_1393 (size=12151) 2024-11-08T18:41:33,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:33,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091353149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:33,156 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2d8b2888d7998b4dc696cf695019721f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:33,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:33,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:33,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:33,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:33,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:33,398 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/6e12d5b0113941eab291093be48651c6 2024-11-08T18:41:33,401 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/996c15fbecb548aa8261bd6dd1a61282 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/996c15fbecb548aa8261bd6dd1a61282 2024-11-08T18:41:33,403 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/996c15fbecb548aa8261bd6dd1a61282, entries=200, sequenceid=208, filesize=38.8 K 2024-11-08T18:41:33,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/d061740139834ba5be1eeecba47312f1 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/d061740139834ba5be1eeecba47312f1 2024-11-08T18:41:33,407 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/d061740139834ba5be1eeecba47312f1, entries=150, sequenceid=208, filesize=11.9 K 2024-11-08T18:41:33,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/6e12d5b0113941eab291093be48651c6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/6e12d5b0113941eab291093be48651c6 2024-11-08T18:41:33,410 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/6e12d5b0113941eab291093be48651c6, entries=150, sequenceid=208, filesize=11.9 K 2024-11-08T18:41:33,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 2d8b2888d7998b4dc696cf695019721f in 1656ms, sequenceid=208, compaction requested=true 2024-11-08T18:41:33,411 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:33,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:33,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:33,411 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:33,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:33,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:33,411 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:33,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:33,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:33,412 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:33,412 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:33,412 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/B is initiating minor compaction (all files) 2024-11-08T18:41:33,412 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/A is initiating minor compaction (all files) 2024-11-08T18:41:33,412 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/A in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,412 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/B in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,412 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/109cfcb8d50a4bb4b9ec1f0c8bdc80bf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/28ab82aabe3549498e77cf241c34c052, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/996c15fbecb548aa8261bd6dd1a61282] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=100.0 K 2024-11-08T18:41:33,412 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0bb7ef03b3e549e59bdc32318da77237, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5a0b5ae1140b4a398fa111c2f23052ab, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/d061740139834ba5be1eeecba47312f1] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=36.0 K 2024-11-08T18:41:33,412 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,412 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/109cfcb8d50a4bb4b9ec1f0c8bdc80bf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/28ab82aabe3549498e77cf241c34c052, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/996c15fbecb548aa8261bd6dd1a61282] 2024-11-08T18:41:33,412 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bb7ef03b3e549e59bdc32318da77237, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731091288860 2024-11-08T18:41:33,412 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 109cfcb8d50a4bb4b9ec1f0c8bdc80bf, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731091288860 2024-11-08T18:41:33,413 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a0b5ae1140b4a398fa111c2f23052ab, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731091288944 2024-11-08T18:41:33,413 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28ab82aabe3549498e77cf241c34c052, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731091288944 2024-11-08T18:41:33,413 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d061740139834ba5be1eeecba47312f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731091291123 2024-11-08T18:41:33,413 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 996c15fbecb548aa8261bd6dd1a61282, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731091291120 2024-11-08T18:41:33,417 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:33,418 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411085e87e8bf4ea6454398ad0321f38c64e1_2d8b2888d7998b4dc696cf695019721f store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:33,419 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#B#compaction#331 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:33,420 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/4515ae8414b94cf3976de472fd7efb09 is 50, key is test_row_0/B:col10/1731091291123/Put/seqid=0 2024-11-08T18:41:33,420 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411085e87e8bf4ea6454398ad0321f38c64e1_2d8b2888d7998b4dc696cf695019721f, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:33,420 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085e87e8bf4ea6454398ad0321f38c64e1_2d8b2888d7998b4dc696cf695019721f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:33,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742219_1395 (size=4469) 2024-11-08T18:41:33,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742218_1394 (size=12663) 2024-11-08T18:41:33,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-08T18:41:33,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,510 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:41:33,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108dc28a8a92f464766a2e2371541ead234_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091291800/Put/seqid=0 2024-11-08T18:41:33,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742220_1396 (size=12304) 2024-11-08T18:41:33,828 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#A#compaction#330 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:33,829 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/17b49698ce8a4e8da8a1e2f89e856a2e is 175, key is test_row_0/A:col10/1731091291123/Put/seqid=0 2024-11-08T18:41:33,833 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/4515ae8414b94cf3976de472fd7efb09 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/4515ae8414b94cf3976de472fd7efb09 2024-11-08T18:41:33,837 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/B of 2d8b2888d7998b4dc696cf695019721f into 4515ae8414b94cf3976de472fd7efb09(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:33,837 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:33,837 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/B, priority=13, startTime=1731091293411; duration=0sec 2024-11-08T18:41:33,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:33,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:B 2024-11-08T18:41:33,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:33,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:33,838 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/C is initiating minor compaction (all files) 2024-11-08T18:41:33,838 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/C in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:33,838 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/1679da2bd1fb42fb903f35ffa4abbfc3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2b33ce6684144a359c6aebb45bda5095, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/6e12d5b0113941eab291093be48651c6] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=36.0 K 2024-11-08T18:41:33,839 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1679da2bd1fb42fb903f35ffa4abbfc3, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731091288860 2024-11-08T18:41:33,839 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b33ce6684144a359c6aebb45bda5095, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731091288944 2024-11-08T18:41:33,839 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e12d5b0113941eab291093be48651c6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731091291123 2024-11-08T18:41:33,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742221_1397 (size=31617) 2024-11-08T18:41:33,846 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#C#compaction#333 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:33,846 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/87bf2ab0da0b4fa0af4a4a721731e243 is 50, key is test_row_0/C:col10/1731091291123/Put/seqid=0 2024-11-08T18:41:33,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742222_1398 (size=12663) 2024-11-08T18:41:33,923 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:33,926 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108dc28a8a92f464766a2e2371541ead234_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108dc28a8a92f464766a2e2371541ead234_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:33,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/908a56294a6244b188edc97744d44fb0, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:33,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/908a56294a6244b188edc97744d44fb0 is 175, key is test_row_0/A:col10/1731091291800/Put/seqid=0 2024-11-08T18:41:33,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742223_1399 (size=31105) 2024-11-08T18:41:33,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:33,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:33,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T18:41:33,983 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:33,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091353977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,984 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:33,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:33,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091353982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:33,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:33,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091353983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,993 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:33,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091353984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:33,993 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091354090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091354090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,097 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091354099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091354099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,106 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,244 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/17b49698ce8a4e8da8a1e2f89e856a2e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/17b49698ce8a4e8da8a1e2f89e856a2e 2024-11-08T18:41:34,247 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/A of 2d8b2888d7998b4dc696cf695019721f into 17b49698ce8a4e8da8a1e2f89e856a2e(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:34,247 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:34,247 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/A, priority=13, startTime=1731091293411; duration=0sec 2024-11-08T18:41:34,247 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:34,247 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:A 2024-11-08T18:41:34,257 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/87bf2ab0da0b4fa0af4a4a721731e243 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/87bf2ab0da0b4fa0af4a4a721731e243 2024-11-08T18:41:34,260 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/C of 2d8b2888d7998b4dc696cf695019721f into 87bf2ab0da0b4fa0af4a4a721731e243(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:34,260 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:34,260 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/C, priority=13, startTime=1731091293411; duration=0sec 2024-11-08T18:41:34,260 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:34,260 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:C 2024-11-08T18:41:34,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091354300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091354309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091354309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091354309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,330 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=232, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/908a56294a6244b188edc97744d44fb0 2024-11-08T18:41:34,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/a44f1d363b084d98973033f37ec6f514 is 50, key is test_row_0/B:col10/1731091291800/Put/seqid=0 2024-11-08T18:41:34,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742224_1400 (size=12151) 2024-11-08T18:41:34,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091354609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,616 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091354619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091354619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091354619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:34,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:34,742 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/a44f1d363b084d98973033f37ec6f514 2024-11-08T18:41:34,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/0fe4bc679360453a9c22741cb71a881f is 50, key is test_row_0/C:col10/1731091291800/Put/seqid=0 2024-11-08T18:41:34,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742225_1401 (size=12151) 2024-11-08T18:41:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091355129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091355129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:35,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:35,137 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091355129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:35,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:35,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091355139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:35,146 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:35,152 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/0fe4bc679360453a9c22741cb71a881f 2024-11-08T18:41:35,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/908a56294a6244b188edc97744d44fb0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/908a56294a6244b188edc97744d44fb0 2024-11-08T18:41:35,159 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/908a56294a6244b188edc97744d44fb0, entries=150, sequenceid=232, filesize=30.4 K 2024-11-08T18:41:35,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/a44f1d363b084d98973033f37ec6f514 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/a44f1d363b084d98973033f37ec6f514 2024-11-08T18:41:35,163 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/a44f1d363b084d98973033f37ec6f514, entries=150, sequenceid=232, filesize=11.9 K 2024-11-08T18:41:35,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/0fe4bc679360453a9c22741cb71a881f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0fe4bc679360453a9c22741cb71a881f 2024-11-08T18:41:35,170 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0fe4bc679360453a9c22741cb71a881f, entries=150, sequenceid=232, filesize=11.9 K 2024-11-08T18:41:35,171 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 2d8b2888d7998b4dc696cf695019721f in 1661ms, sequenceid=232, compaction requested=false 2024-11-08T18:41:35,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:35,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:35,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-08T18:41:35,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-08T18:41:35,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-08T18:41:35,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3410 sec 2024-11-08T18:41:35,176 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.3450 sec 2024-11-08T18:41:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-08T18:41:35,990 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:35,990 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-08T18:41:35,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T18:41:35,992 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:35,992 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:35,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T18:41:36,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:36,144 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:36,144 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:41:36,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T18:41:36,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:36,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,154 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110812bdefe7e94445a6886676ea142b95e2_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091293982/Put/seqid=0 2024-11-08T18:41:36,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742226_1402 (size=14794) 2024-11-08T18:41:36,159 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:36,162 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110812bdefe7e94445a6886676ea142b95e2_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110812bdefe7e94445a6886676ea142b95e2_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:36,163 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/8ea4422920cd4b01a37775f23dad9c6d, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:36,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/8ea4422920cd4b01a37775f23dad9c6d is 175, key is test_row_0/A:col10/1731091293982/Put/seqid=0 2024-11-08T18:41:36,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742227_1403 (size=39749) 2024-11-08T18:41:36,168 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=249, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/8ea4422920cd4b01a37775f23dad9c6d 2024-11-08T18:41:36,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/0cedd15a03294b9a84b557cabc9abab6 is 50, key is test_row_0/B:col10/1731091293982/Put/seqid=0 2024-11-08T18:41:36,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742228_1404 (size=12151) 2024-11-08T18:41:36,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091356188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,191 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091356189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091356190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,198 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091356191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T18:41:36,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:36,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,298 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091356300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091356300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,304 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T18:41:36,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091356309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091356309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T18:41:36,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:36,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091356510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091356510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,514 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091356519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,523 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091356529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,532 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,581 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/0cedd15a03294b9a84b557cabc9abab6 2024-11-08T18:41:36,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T18:41:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:36,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,603 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T18:41:36,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/e19b38e0260c4267a0be57db4ab633fd is 50, key is test_row_0/C:col10/1731091293982/Put/seqid=0 2024-11-08T18:41:36,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742229_1405 (size=12151) 2024-11-08T18:41:36,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T18:41:36,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091356819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091356819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091356829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:36,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091356839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:36,842 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:36,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:36,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T18:41:36,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:36,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:36,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:37,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/e19b38e0260c4267a0be57db4ab633fd 2024-11-08T18:41:37,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/8ea4422920cd4b01a37775f23dad9c6d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/8ea4422920cd4b01a37775f23dad9c6d 2024-11-08T18:41:37,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/8ea4422920cd4b01a37775f23dad9c6d, entries=200, sequenceid=249, filesize=38.8 K 2024-11-08T18:41:37,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/0cedd15a03294b9a84b557cabc9abab6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0cedd15a03294b9a84b557cabc9abab6 2024-11-08T18:41:37,045 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0cedd15a03294b9a84b557cabc9abab6, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T18:41:37,045 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/e19b38e0260c4267a0be57db4ab633fd as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/e19b38e0260c4267a0be57db4ab633fd 2024-11-08T18:41:37,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/e19b38e0260c4267a0be57db4ab633fd, entries=150, sequenceid=249, filesize=11.9 K 2024-11-08T18:41:37,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 2d8b2888d7998b4dc696cf695019721f in 905ms, sequenceid=249, compaction requested=true 2024-11-08T18:41:37,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:37,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:37,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:37,050 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:37,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:37,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:37,050 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:37,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:37,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:37,051 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102471 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:37,051 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:37,051 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/B is initiating minor compaction (all files) 2024-11-08T18:41:37,051 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/A is initiating minor compaction (all files) 2024-11-08T18:41:37,051 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/A in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:37,051 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/B in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:37,051 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/4515ae8414b94cf3976de472fd7efb09, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/a44f1d363b084d98973033f37ec6f514, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0cedd15a03294b9a84b557cabc9abab6] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=36.1 K 2024-11-08T18:41:37,051 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/17b49698ce8a4e8da8a1e2f89e856a2e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/908a56294a6244b188edc97744d44fb0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/8ea4422920cd4b01a37775f23dad9c6d] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=100.1 K 2024-11-08T18:41:37,051 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:37,051 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/17b49698ce8a4e8da8a1e2f89e856a2e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/908a56294a6244b188edc97744d44fb0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/8ea4422920cd4b01a37775f23dad9c6d] 2024-11-08T18:41:37,051 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4515ae8414b94cf3976de472fd7efb09, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731091291123 2024-11-08T18:41:37,052 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17b49698ce8a4e8da8a1e2f89e856a2e, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731091291123 2024-11-08T18:41:37,052 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting a44f1d363b084d98973033f37ec6f514, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091291792 2024-11-08T18:41:37,052 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 908a56294a6244b188edc97744d44fb0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091291792 2024-11-08T18:41:37,052 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cedd15a03294b9a84b557cabc9abab6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091293982 2024-11-08T18:41:37,052 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ea4422920cd4b01a37775f23dad9c6d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091293981 2024-11-08T18:41:37,057 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:37,061 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#B#compaction#340 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:37,061 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/5e34dab034ce48ff94eda97a6a16932c is 50, key is test_row_0/B:col10/1731091293982/Put/seqid=0 2024-11-08T18:41:37,063 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108b07801dd3a714fcaaaad38ff519b3cd5_2d8b2888d7998b4dc696cf695019721f store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:37,065 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108b07801dd3a714fcaaaad38ff519b3cd5_2d8b2888d7998b4dc696cf695019721f, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:37,065 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108b07801dd3a714fcaaaad38ff519b3cd5_2d8b2888d7998b4dc696cf695019721f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:37,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742230_1406 (size=12765) 2024-11-08T18:41:37,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742231_1407 (size=4469) 2024-11-08T18:41:37,080 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#A#compaction#339 average throughput is 1.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:37,081 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/558d6641ac7c435a90b6edde0c4ee362 is 175, key is test_row_0/A:col10/1731091293982/Put/seqid=0 2024-11-08T18:41:37,082 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/5e34dab034ce48ff94eda97a6a16932c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5e34dab034ce48ff94eda97a6a16932c 2024-11-08T18:41:37,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-08T18:41:37,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:37,084 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T18:41:37,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742232_1408 (size=31719) 2024-11-08T18:41:37,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110828ab128524a94040bd07e0eaf2f6eb12_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091296189/Put/seqid=0 2024-11-08T18:41:37,094 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/B of 2d8b2888d7998b4dc696cf695019721f into 5e34dab034ce48ff94eda97a6a16932c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:37,094 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:37,094 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/B, priority=13, startTime=1731091297050; duration=0sec 2024-11-08T18:41:37,094 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:37,094 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:B 2024-11-08T18:41:37,094 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:37,095 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36965 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:37,095 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/C is initiating minor compaction (all files) 2024-11-08T18:41:37,095 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/C in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:37,095 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/87bf2ab0da0b4fa0af4a4a721731e243, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0fe4bc679360453a9c22741cb71a881f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/e19b38e0260c4267a0be57db4ab633fd] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=36.1 K 2024-11-08T18:41:37,096 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 87bf2ab0da0b4fa0af4a4a721731e243, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731091291123 2024-11-08T18:41:37,096 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fe4bc679360453a9c22741cb71a881f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091291792 2024-11-08T18:41:37,096 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting e19b38e0260c4267a0be57db4ab633fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091293982 2024-11-08T18:41:37,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742233_1409 (size=12454) 2024-11-08T18:41:37,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:37,102 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110828ab128524a94040bd07e0eaf2f6eb12_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110828ab128524a94040bd07e0eaf2f6eb12_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:37,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/ed5762fa24ed41fb9274a662fc85b4dc, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:37,103 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:37,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/ed5762fa24ed41fb9274a662fc85b4dc is 175, key is test_row_0/A:col10/1731091296189/Put/seqid=0 2024-11-08T18:41:37,103 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/5c14503942d8426fb07fbdebd703e0e4 is 50, key is test_row_0/C:col10/1731091293982/Put/seqid=0 2024-11-08T18:41:37,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742234_1410 (size=31255) 2024-11-08T18:41:37,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742235_1411 (size=12765) 2024-11-08T18:41:37,116 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/5c14503942d8426fb07fbdebd703e0e4 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/5c14503942d8426fb07fbdebd703e0e4 2024-11-08T18:41:37,120 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/C of 2d8b2888d7998b4dc696cf695019721f into 5c14503942d8426fb07fbdebd703e0e4(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:37,120 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:37,120 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/C, priority=13, startTime=1731091297050; duration=0sec 2024-11-08T18:41:37,120 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:37,120 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:C 2024-11-08T18:41:37,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T18:41:37,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. as already flushing 2024-11-08T18:41:37,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:37,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:37,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091357230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:37,334 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:37,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091357329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:37,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:37,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091357339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:37,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36232 deadline: 1731091357339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091357340, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,344 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:37,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:37,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36260 deadline: 1731091357349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:37,490 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/558d6641ac7c435a90b6edde0c4ee362 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/558d6641ac7c435a90b6edde0c4ee362 2024-11-08T18:41:37,494 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/A of 2d8b2888d7998b4dc696cf695019721f into 558d6641ac7c435a90b6edde0c4ee362(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:37,494 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:37,494 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/A, priority=13, startTime=1731091297050; duration=0sec 2024-11-08T18:41:37,494 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:37,494 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:A 2024-11-08T18:41:37,509 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=271, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/ed5762fa24ed41fb9274a662fc85b4dc 2024-11-08T18:41:37,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/458187fc7cd7489e8e84bf8f7c892dfb is 50, key is test_row_0/B:col10/1731091296189/Put/seqid=0 2024-11-08T18:41:37,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742236_1412 (size=12301) 2024-11-08T18:41:37,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:37,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091357549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:37,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36246 deadline: 1731091357859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:37,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:37,918 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/458187fc7cd7489e8e84bf8f7c892dfb 2024-11-08T18:41:37,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/dea42a46a0f64660be91b7187de9f322 is 50, key is test_row_0/C:col10/1731091296189/Put/seqid=0 2024-11-08T18:41:37,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742237_1413 (size=12301) 2024-11-08T18:41:38,047 INFO [Thread-1537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1537. 2024-11-08T18:41:38,047 DEBUG [Thread-1537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:38,047 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,047 DEBUG [Thread-1537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,047 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:38,049 INFO [Thread-1533 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1533. 2024-11-08T18:41:38,049 DEBUG [Thread-1533 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:38,049 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,049 DEBUG [Thread-1533 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:38,050 INFO [Thread-1539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1539. 2024-11-08T18:41:38,050 DEBUG [Thread-1539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:38,050 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,050 DEBUG [Thread-1539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,050 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:38,055 INFO [Thread-1541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1541. 2024-11-08T18:41:38,055 DEBUG [Thread-1541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:38,055 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,055 DEBUG [Thread-1541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,055 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:38,057 INFO [Thread-1535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1535. 2024-11-08T18:41:38,057 DEBUG [Thread-1535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:38,057 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,057 DEBUG [Thread-1535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,057 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T18:41:38,328 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/dea42a46a0f64660be91b7187de9f322 2024-11-08T18:41:38,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/ed5762fa24ed41fb9274a662fc85b4dc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/ed5762fa24ed41fb9274a662fc85b4dc 2024-11-08T18:41:38,342 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/ed5762fa24ed41fb9274a662fc85b4dc, entries=150, sequenceid=271, filesize=30.5 K 2024-11-08T18:41:38,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/458187fc7cd7489e8e84bf8f7c892dfb as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/458187fc7cd7489e8e84bf8f7c892dfb 2024-11-08T18:41:38,348 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/458187fc7cd7489e8e84bf8f7c892dfb, entries=150, sequenceid=271, filesize=12.0 K 2024-11-08T18:41:38,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/dea42a46a0f64660be91b7187de9f322 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/dea42a46a0f64660be91b7187de9f322 2024-11-08T18:41:38,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:38,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36224 deadline: 1731091358350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36218 deadline: 1731091358350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:38,350 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2d8b2888d7998b4dc696cf695019721f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:38,350 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in 2d8b2888d7998b4dc696cf695019721f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:38,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:41:38,352 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/dea42a46a0f64660be91b7187de9f322, entries=150, sequenceid=271, filesize=12.0 K 2024-11-08T18:41:38,353 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 2d8b2888d7998b4dc696cf695019721f in 1269ms, sequenceid=271, compaction requested=false 2024-11-08T18:41:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:38,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-08T18:41:38,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-08T18:41:38,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-08T18:41:38,356 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3620 sec 2024-11-08T18:41:38,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.3660 sec 2024-11-08T18:41:38,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:38,360 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:38,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T18:41:38,360 INFO [Thread-1530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1530. 2024-11-08T18:41:38,360 INFO [Thread-1528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1528. 2024-11-08T18:41:38,360 DEBUG [Thread-1528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:38,360 DEBUG [Thread-1530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:38,360 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,360 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,360 DEBUG [Thread-1528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,360 DEBUG [Thread-1530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:38,360 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:38,365 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110818becba9b17c4c7fa3691f884b064940_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091297228/Put/seqid=0 2024-11-08T18:41:38,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742238_1414 (size=12454) 2024-11-08T18:41:38,381 INFO [Thread-1522 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1522. 2024-11-08T18:41:38,381 DEBUG [Thread-1522 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:38,381 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,381 DEBUG [Thread-1522 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:38,381 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:38,770 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:38,779 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110818becba9b17c4c7fa3691f884b064940_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110818becba9b17c4c7fa3691f884b064940_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:38,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/2c8141643e1c4df4916e2065c6c604d4, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:38,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/2c8141643e1c4df4916e2065c6c604d4 is 175, key is test_row_0/A:col10/1731091297228/Put/seqid=0 2024-11-08T18:41:38,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742239_1415 (size=31255) 2024-11-08T18:41:39,188 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=290, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/2c8141643e1c4df4916e2065c6c604d4 2024-11-08T18:41:39,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/aee21ad0d29f40ba8d7d54a84749f25a is 50, key is test_row_0/B:col10/1731091297228/Put/seqid=0 2024-11-08T18:41:39,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742240_1416 (size=12301) 2024-11-08T18:41:39,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/aee21ad0d29f40ba8d7d54a84749f25a 2024-11-08T18:41:39,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f2afebe1436a43d78e636a151b3eab27 is 50, key is test_row_0/C:col10/1731091297228/Put/seqid=0 2024-11-08T18:41:39,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742241_1417 (size=12301) 2024-11-08T18:41:40,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f2afebe1436a43d78e636a151b3eab27 2024-11-08T18:41:40,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/2c8141643e1c4df4916e2065c6c604d4 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2c8141643e1c4df4916e2065c6c604d4 2024-11-08T18:41:40,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2c8141643e1c4df4916e2065c6c604d4, entries=150, sequenceid=290, filesize=30.5 K 2024-11-08T18:41:40,039 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/aee21ad0d29f40ba8d7d54a84749f25a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/aee21ad0d29f40ba8d7d54a84749f25a 2024-11-08T18:41:40,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/aee21ad0d29f40ba8d7d54a84749f25a, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T18:41:40,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/f2afebe1436a43d78e636a151b3eab27 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f2afebe1436a43d78e636a151b3eab27 2024-11-08T18:41:40,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f2afebe1436a43d78e636a151b3eab27, entries=150, sequenceid=290, filesize=12.0 K 2024-11-08T18:41:40,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=6.71 KB/6870 for 2d8b2888d7998b4dc696cf695019721f in 1686ms, sequenceid=290, compaction requested=true 2024-11-08T18:41:40,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:40,047 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2d8b2888d7998b4dc696cf695019721f:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:40,047 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:40,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:40,048 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:40,048 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/A is initiating minor compaction (all files) 2024-11-08T18:41:40,048 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/A in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:40,048 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:40,048 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/B is initiating minor compaction (all files) 2024-11-08T18:41:40,048 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/558d6641ac7c435a90b6edde0c4ee362, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/ed5762fa24ed41fb9274a662fc85b4dc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2c8141643e1c4df4916e2065c6c604d4] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=92.0 K 2024-11-08T18:41:40,048 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/B in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:40,048 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:40,048 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5e34dab034ce48ff94eda97a6a16932c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/458187fc7cd7489e8e84bf8f7c892dfb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/aee21ad0d29f40ba8d7d54a84749f25a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=36.5 K 2024-11-08T18:41:40,048 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/558d6641ac7c435a90b6edde0c4ee362, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/ed5762fa24ed41fb9274a662fc85b4dc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2c8141643e1c4df4916e2065c6c604d4] 2024-11-08T18:41:40,048 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 558d6641ac7c435a90b6edde0c4ee362, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091293982 2024-11-08T18:41:40,048 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e34dab034ce48ff94eda97a6a16932c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091293982 2024-11-08T18:41:40,049 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed5762fa24ed41fb9274a662fc85b4dc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731091296181 2024-11-08T18:41:40,049 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 458187fc7cd7489e8e84bf8f7c892dfb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731091296181 2024-11-08T18:41:40,049 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c8141643e1c4df4916e2065c6c604d4, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731091297228 2024-11-08T18:41:40,049 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting aee21ad0d29f40ba8d7d54a84749f25a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731091297228 2024-11-08T18:41:40,059 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#B#compaction#348 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:40,060 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/46150377e5ad40948153fa246fd9b35a is 50, key is test_row_0/B:col10/1731091297228/Put/seqid=0 2024-11-08T18:41:40,061 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:40,063 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411088e8ec73a69a2402dacb675323b1b56c0_2d8b2888d7998b4dc696cf695019721f store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:40,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742242_1418 (size=13017) 2024-11-08T18:41:40,086 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411088e8ec73a69a2402dacb675323b1b56c0_2d8b2888d7998b4dc696cf695019721f, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:40,086 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088e8ec73a69a2402dacb675323b1b56c0_2d8b2888d7998b4dc696cf695019721f because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:40,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742243_1419 (size=4469) 2024-11-08T18:41:40,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-08T18:41:40,150 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:40,361 INFO [Thread-1524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1524. 2024-11-08T18:41:40,362 DEBUG [Thread-1524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:40,362 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:40,362 DEBUG [Thread-1524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:40,362 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:40,372 INFO [Thread-1526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1526. 2024-11-08T18:41:40,372 DEBUG [Thread-1526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:41:40,372 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:40,372 DEBUG [Thread-1526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:40,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1499 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4490 rows 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1502 2024-11-08T18:41:40,372 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4499 rows 2024-11-08T18:41:40,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1491 2024-11-08T18:41:40,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4472 rows 2024-11-08T18:41:40,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1499 2024-11-08T18:41:40,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4493 rows 2024-11-08T18:41:40,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1489 2024-11-08T18:41:40,373 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4460 rows 2024-11-08T18:41:40,373 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T18:41:40,373 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:41:40,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:40,373 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:40,373 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:41:40,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T18:41:40,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:40,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T18:41:40,380 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091300380"}]},"ts":"1731091300380"} 2024-11-08T18:41:40,383 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T18:41:40,383 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T18:41:40,384 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:41:40,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, UNASSIGN}] 2024-11-08T18:41:40,387 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, UNASSIGN 2024-11-08T18:41:40,388 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=2d8b2888d7998b4dc696cf695019721f, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:40,390 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=111, ppid=110, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, UNASSIGN because future has completed 2024-11-08T18:41:40,390 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:41:40,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:41:40,476 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/46150377e5ad40948153fa246fd9b35a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/46150377e5ad40948153fa246fd9b35a 2024-11-08T18:41:40,484 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/B of 2d8b2888d7998b4dc696cf695019721f into 46150377e5ad40948153fa246fd9b35a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:40,484 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:40,484 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/B, priority=13, startTime=1731091300047; duration=0sec 2024-11-08T18:41:40,484 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:40,484 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:B 2024-11-08T18:41:40,484 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:40,486 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:40,486 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 2d8b2888d7998b4dc696cf695019721f/C is initiating minor compaction (all files) 2024-11-08T18:41:40,486 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2d8b2888d7998b4dc696cf695019721f/C in TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:40,486 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/5c14503942d8426fb07fbdebd703e0e4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/dea42a46a0f64660be91b7187de9f322, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f2afebe1436a43d78e636a151b3eab27] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp, totalSize=36.5 K 2024-11-08T18:41:40,487 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c14503942d8426fb07fbdebd703e0e4, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731091293982 2024-11-08T18:41:40,487 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting dea42a46a0f64660be91b7187de9f322, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731091296181 2024-11-08T18:41:40,488 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f2afebe1436a43d78e636a151b3eab27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731091297228 2024-11-08T18:41:40,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T18:41:40,491 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#A#compaction#349 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:40,492 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/f955cdee4c4c44f5b8c287f5b233fbfa is 175, key is test_row_0/A:col10/1731091297228/Put/seqid=0 2024-11-08T18:41:40,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742244_1420 (size=31971) 2024-11-08T18:41:40,497 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2d8b2888d7998b4dc696cf695019721f#C#compaction#350 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:40,498 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/c65bf870f765406a8787f1b8718530c2 is 50, key is test_row_0/C:col10/1731091297228/Put/seqid=0 2024-11-08T18:41:40,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742245_1421 (size=13017) 2024-11-08T18:41:40,543 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(122): Close 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:40,544 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:41:40,544 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1722): Closing 2d8b2888d7998b4dc696cf695019721f, disabling compactions & flushes 2024-11-08T18:41:40,544 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1993): waiting for 2 compactions to complete for region TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T18:41:40,907 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/f955cdee4c4c44f5b8c287f5b233fbfa as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f955cdee4c4c44f5b8c287f5b233fbfa 2024-11-08T18:41:40,909 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/c65bf870f765406a8787f1b8718530c2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/c65bf870f765406a8787f1b8718530c2 2024-11-08T18:41:40,913 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/A of 2d8b2888d7998b4dc696cf695019721f into f955cdee4c4c44f5b8c287f5b233fbfa(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:40,913 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:40,913 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/A, priority=13, startTime=1731091300047; duration=0sec 2024-11-08T18:41:40,913 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:40,913 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:A 2024-11-08T18:41:40,915 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2d8b2888d7998b4dc696cf695019721f/C of 2d8b2888d7998b4dc696cf695019721f into c65bf870f765406a8787f1b8718530c2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:40,915 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2d8b2888d7998b4dc696cf695019721f: 2024-11-08T18:41:40,915 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., storeName=2d8b2888d7998b4dc696cf695019721f/C, priority=13, startTime=1731091300047; duration=0sec 2024-11-08T18:41:40,915 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:40,915 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:40,915 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:40,915 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. after waiting 0 ms 2024-11-08T18:41:40,915 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2d8b2888d7998b4dc696cf695019721f:C 2024-11-08T18:41:40,915 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:40,915 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(2902): Flushing 2d8b2888d7998b4dc696cf695019721f 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-08T18:41:40,921 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083b9816ee100143148def5bf3b9327de1_2d8b2888d7998b4dc696cf695019721f is 50, key is test_row_0/A:col10/1731091300370/Put/seqid=0 2024-11-08T18:41:40,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742246_1422 (size=9914) 2024-11-08T18:41:41,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T18:41:41,325 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:41,333 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411083b9816ee100143148def5bf3b9327de1_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b9816ee100143148def5bf3b9327de1_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:41,334 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/0d83c30a5adf40cfaad0209ce77527e0, store: [table=TestAcidGuarantees family=A region=2d8b2888d7998b4dc696cf695019721f] 2024-11-08T18:41:41,335 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/0d83c30a5adf40cfaad0209ce77527e0 is 175, key is test_row_0/A:col10/1731091300370/Put/seqid=0 2024-11-08T18:41:41,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742247_1423 (size=22561) 2024-11-08T18:41:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T18:41:41,638 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T18:41:41,740 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=299, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/0d83c30a5adf40cfaad0209ce77527e0 2024-11-08T18:41:41,785 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/7174dc628f2e4aa1adfbfce4968b708a is 50, key is test_row_0/B:col10/1731091300370/Put/seqid=0 2024-11-08T18:41:41,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742248_1424 (size=9857) 2024-11-08T18:41:42,192 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/7174dc628f2e4aa1adfbfce4968b708a 2024-11-08T18:41:42,201 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/43514520294d4aab80d3385dff3f1436 is 50, key is test_row_0/C:col10/1731091300370/Put/seqid=0 2024-11-08T18:41:42,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742249_1425 (size=9857) 2024-11-08T18:41:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T18:41:42,606 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/43514520294d4aab80d3385dff3f1436 2024-11-08T18:41:42,616 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/A/0d83c30a5adf40cfaad0209ce77527e0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0d83c30a5adf40cfaad0209ce77527e0 2024-11-08T18:41:42,620 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0d83c30a5adf40cfaad0209ce77527e0, entries=100, sequenceid=299, filesize=22.0 K 2024-11-08T18:41:42,621 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/B/7174dc628f2e4aa1adfbfce4968b708a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/7174dc628f2e4aa1adfbfce4968b708a 2024-11-08T18:41:42,626 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/7174dc628f2e4aa1adfbfce4968b708a, entries=100, sequenceid=299, filesize=9.6 K 2024-11-08T18:41:42,627 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/.tmp/C/43514520294d4aab80d3385dff3f1436 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/43514520294d4aab80d3385dff3f1436 2024-11-08T18:41:42,631 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/43514520294d4aab80d3385dff3f1436, entries=100, sequenceid=299, filesize=9.6 K 2024-11-08T18:41:42,632 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 2d8b2888d7998b4dc696cf695019721f in 1717ms, sequenceid=299, compaction requested=false 2024-11-08T18:41:42,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/64048c569a1d442f9c94127f09b1dd46, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/d8a1dfb8b5ca4b758ea7cef03a296a27, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2d49708a073a4e31b832387553645ed9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/02a9644186ee48adb4d34dec6270ca5c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/b01f2ae840364f6db2d04cd3b0374bca, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f6bc249162e84a5caaa1a19454740d67, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/fa3b9c779066429e9327b12419b2bc7a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/7a0b0cf0079445029c0c712d110ad72d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/3cd8f8a872c741a2a2bc44245d31ad61, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/6a3af0f13fb544cdbd683a978bd7056f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0e47800e7a2a49dfaa3ec0e70498e8ea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/233694830ca242f3877ecb9a5c422b69, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/109cfcb8d50a4bb4b9ec1f0c8bdc80bf, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/28ab82aabe3549498e77cf241c34c052, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/996c15fbecb548aa8261bd6dd1a61282, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/17b49698ce8a4e8da8a1e2f89e856a2e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/908a56294a6244b188edc97744d44fb0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/8ea4422920cd4b01a37775f23dad9c6d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/558d6641ac7c435a90b6edde0c4ee362, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/ed5762fa24ed41fb9274a662fc85b4dc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2c8141643e1c4df4916e2065c6c604d4] to archive 2024-11-08T18:41:42,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:41:42,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/64048c569a1d442f9c94127f09b1dd46 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/64048c569a1d442f9c94127f09b1dd46 2024-11-08T18:41:42,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/d8a1dfb8b5ca4b758ea7cef03a296a27 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/d8a1dfb8b5ca4b758ea7cef03a296a27 2024-11-08T18:41:42,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2d49708a073a4e31b832387553645ed9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2d49708a073a4e31b832387553645ed9 2024-11-08T18:41:42,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/02a9644186ee48adb4d34dec6270ca5c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/02a9644186ee48adb4d34dec6270ca5c 2024-11-08T18:41:42,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/b01f2ae840364f6db2d04cd3b0374bca to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/b01f2ae840364f6db2d04cd3b0374bca 2024-11-08T18:41:42,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f6bc249162e84a5caaa1a19454740d67 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f6bc249162e84a5caaa1a19454740d67 2024-11-08T18:41:42,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/fa3b9c779066429e9327b12419b2bc7a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/fa3b9c779066429e9327b12419b2bc7a 2024-11-08T18:41:42,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/7a0b0cf0079445029c0c712d110ad72d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/7a0b0cf0079445029c0c712d110ad72d 2024-11-08T18:41:42,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/3cd8f8a872c741a2a2bc44245d31ad61 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/3cd8f8a872c741a2a2bc44245d31ad61 2024-11-08T18:41:42,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/6a3af0f13fb544cdbd683a978bd7056f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/6a3af0f13fb544cdbd683a978bd7056f 2024-11-08T18:41:42,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0e47800e7a2a49dfaa3ec0e70498e8ea to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0e47800e7a2a49dfaa3ec0e70498e8ea 2024-11-08T18:41:42,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/233694830ca242f3877ecb9a5c422b69 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/233694830ca242f3877ecb9a5c422b69 2024-11-08T18:41:42,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/109cfcb8d50a4bb4b9ec1f0c8bdc80bf to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/109cfcb8d50a4bb4b9ec1f0c8bdc80bf 2024-11-08T18:41:42,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/28ab82aabe3549498e77cf241c34c052 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/28ab82aabe3549498e77cf241c34c052 2024-11-08T18:41:42,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/996c15fbecb548aa8261bd6dd1a61282 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/996c15fbecb548aa8261bd6dd1a61282 2024-11-08T18:41:42,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/17b49698ce8a4e8da8a1e2f89e856a2e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/17b49698ce8a4e8da8a1e2f89e856a2e 2024-11-08T18:41:42,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/908a56294a6244b188edc97744d44fb0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/908a56294a6244b188edc97744d44fb0 2024-11-08T18:41:42,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/8ea4422920cd4b01a37775f23dad9c6d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/8ea4422920cd4b01a37775f23dad9c6d 2024-11-08T18:41:42,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/558d6641ac7c435a90b6edde0c4ee362 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/558d6641ac7c435a90b6edde0c4ee362 2024-11-08T18:41:42,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/ed5762fa24ed41fb9274a662fc85b4dc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/ed5762fa24ed41fb9274a662fc85b4dc 2024-11-08T18:41:42,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2c8141643e1c4df4916e2065c6c604d4 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/2c8141643e1c4df4916e2065c6c604d4 2024-11-08T18:41:42,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/1745800b2d6e40b9b7436bdf77454a61, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/38877608cdea41feb848c08fdd8e3b3b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/8db0fc04e6d44b70876fc3baf9cb3f0d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/9995d40298714ad19b4cfffdb5bd8bac, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/73e7e326ede94756a49eed9b2460dde6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/acf6b83414f84f9daf929241b3b570f2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f64e0806e0cc46b78a0721ec4932b922, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/28ee43d436a54b1c807aaf210bdd6be3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/06546d247b3f4e7b819221ec66b7c59e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f0966c276510487fa0ecd783ec9bc298, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/b1d96ad5012442e29de664515882dcde, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0bb7ef03b3e549e59bdc32318da77237, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/c374421a10734c478e59839200814a39, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5a0b5ae1140b4a398fa111c2f23052ab, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/4515ae8414b94cf3976de472fd7efb09, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/d061740139834ba5be1eeecba47312f1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/a44f1d363b084d98973033f37ec6f514, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5e34dab034ce48ff94eda97a6a16932c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0cedd15a03294b9a84b557cabc9abab6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/458187fc7cd7489e8e84bf8f7c892dfb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/aee21ad0d29f40ba8d7d54a84749f25a] to archive 2024-11-08T18:41:42,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:41:42,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/1745800b2d6e40b9b7436bdf77454a61 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/1745800b2d6e40b9b7436bdf77454a61 2024-11-08T18:41:42,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/38877608cdea41feb848c08fdd8e3b3b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/38877608cdea41feb848c08fdd8e3b3b 2024-11-08T18:41:42,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/8db0fc04e6d44b70876fc3baf9cb3f0d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/8db0fc04e6d44b70876fc3baf9cb3f0d 2024-11-08T18:41:42,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/9995d40298714ad19b4cfffdb5bd8bac to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/9995d40298714ad19b4cfffdb5bd8bac 2024-11-08T18:41:42,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/73e7e326ede94756a49eed9b2460dde6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/73e7e326ede94756a49eed9b2460dde6 2024-11-08T18:41:42,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/acf6b83414f84f9daf929241b3b570f2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/acf6b83414f84f9daf929241b3b570f2 2024-11-08T18:41:42,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f64e0806e0cc46b78a0721ec4932b922 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f64e0806e0cc46b78a0721ec4932b922 2024-11-08T18:41:42,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/28ee43d436a54b1c807aaf210bdd6be3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/28ee43d436a54b1c807aaf210bdd6be3 2024-11-08T18:41:42,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/06546d247b3f4e7b819221ec66b7c59e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/06546d247b3f4e7b819221ec66b7c59e 2024-11-08T18:41:42,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f0966c276510487fa0ecd783ec9bc298 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/f0966c276510487fa0ecd783ec9bc298 2024-11-08T18:41:42,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/b1d96ad5012442e29de664515882dcde to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/b1d96ad5012442e29de664515882dcde 2024-11-08T18:41:42,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0bb7ef03b3e549e59bdc32318da77237 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0bb7ef03b3e549e59bdc32318da77237 2024-11-08T18:41:42,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/c374421a10734c478e59839200814a39 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/c374421a10734c478e59839200814a39 2024-11-08T18:41:42,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5a0b5ae1140b4a398fa111c2f23052ab to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5a0b5ae1140b4a398fa111c2f23052ab 2024-11-08T18:41:42,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/4515ae8414b94cf3976de472fd7efb09 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/4515ae8414b94cf3976de472fd7efb09 2024-11-08T18:41:42,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/d061740139834ba5be1eeecba47312f1 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/d061740139834ba5be1eeecba47312f1 2024-11-08T18:41:42,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/a44f1d363b084d98973033f37ec6f514 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/a44f1d363b084d98973033f37ec6f514 2024-11-08T18:41:42,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5e34dab034ce48ff94eda97a6a16932c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/5e34dab034ce48ff94eda97a6a16932c 2024-11-08T18:41:42,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0cedd15a03294b9a84b557cabc9abab6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/0cedd15a03294b9a84b557cabc9abab6 2024-11-08T18:41:42,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/458187fc7cd7489e8e84bf8f7c892dfb to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/458187fc7cd7489e8e84bf8f7c892dfb 2024-11-08T18:41:42,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/aee21ad0d29f40ba8d7d54a84749f25a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/aee21ad0d29f40ba8d7d54a84749f25a 2024-11-08T18:41:42,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/7ebf9f9c3291475a80ac993d5b1ae7c5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f20cdc1b249a441eaded75a2d0553224, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2d7b239e914c4554ab8c5ee3143563ea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8b509e24adc94ce7bc9c895454132a91, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f06a6b6080724816a5650baac0451843, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/9b34d28b916941b8b9af7f762a1f1d00, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/cebc20b892a64a0381b56ee2e9beb47b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2816bf7cbccd438cb6be681c5d8f3c14, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0360b5bf69ff49e984d9f11c84c8cf65, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/098e4fd03a8e44608723cb9ff8136d7d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8278af79100c40dbbeed5b20a114aee2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/1679da2bd1fb42fb903f35ffa4abbfc3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/3419aef58c1e43e38b9b8f48bf198373, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2b33ce6684144a359c6aebb45bda5095, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/87bf2ab0da0b4fa0af4a4a721731e243, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/6e12d5b0113941eab291093be48651c6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0fe4bc679360453a9c22741cb71a881f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/5c14503942d8426fb07fbdebd703e0e4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/e19b38e0260c4267a0be57db4ab633fd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/dea42a46a0f64660be91b7187de9f322, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f2afebe1436a43d78e636a151b3eab27] to archive 2024-11-08T18:41:42,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:41:42,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/7ebf9f9c3291475a80ac993d5b1ae7c5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/7ebf9f9c3291475a80ac993d5b1ae7c5 2024-11-08T18:41:42,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f20cdc1b249a441eaded75a2d0553224 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f20cdc1b249a441eaded75a2d0553224 2024-11-08T18:41:42,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2d7b239e914c4554ab8c5ee3143563ea to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2d7b239e914c4554ab8c5ee3143563ea 2024-11-08T18:41:42,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8b509e24adc94ce7bc9c895454132a91 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8b509e24adc94ce7bc9c895454132a91 2024-11-08T18:41:42,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f06a6b6080724816a5650baac0451843 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f06a6b6080724816a5650baac0451843 2024-11-08T18:41:42,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/9b34d28b916941b8b9af7f762a1f1d00 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/9b34d28b916941b8b9af7f762a1f1d00 2024-11-08T18:41:42,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/cebc20b892a64a0381b56ee2e9beb47b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/cebc20b892a64a0381b56ee2e9beb47b 2024-11-08T18:41:42,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2816bf7cbccd438cb6be681c5d8f3c14 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2816bf7cbccd438cb6be681c5d8f3c14 2024-11-08T18:41:42,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0360b5bf69ff49e984d9f11c84c8cf65 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0360b5bf69ff49e984d9f11c84c8cf65 2024-11-08T18:41:42,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/098e4fd03a8e44608723cb9ff8136d7d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/098e4fd03a8e44608723cb9ff8136d7d 2024-11-08T18:41:42,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8278af79100c40dbbeed5b20a114aee2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/8278af79100c40dbbeed5b20a114aee2 2024-11-08T18:41:42,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/1679da2bd1fb42fb903f35ffa4abbfc3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/1679da2bd1fb42fb903f35ffa4abbfc3 2024-11-08T18:41:42,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/3419aef58c1e43e38b9b8f48bf198373 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/3419aef58c1e43e38b9b8f48bf198373 2024-11-08T18:41:42,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2b33ce6684144a359c6aebb45bda5095 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/2b33ce6684144a359c6aebb45bda5095 2024-11-08T18:41:42,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/87bf2ab0da0b4fa0af4a4a721731e243 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/87bf2ab0da0b4fa0af4a4a721731e243 2024-11-08T18:41:42,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/6e12d5b0113941eab291093be48651c6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/6e12d5b0113941eab291093be48651c6 2024-11-08T18:41:42,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0fe4bc679360453a9c22741cb71a881f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/0fe4bc679360453a9c22741cb71a881f 2024-11-08T18:41:42,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/5c14503942d8426fb07fbdebd703e0e4 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/5c14503942d8426fb07fbdebd703e0e4 2024-11-08T18:41:42,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/e19b38e0260c4267a0be57db4ab633fd to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/e19b38e0260c4267a0be57db4ab633fd 2024-11-08T18:41:42,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/dea42a46a0f64660be91b7187de9f322 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/dea42a46a0f64660be91b7187de9f322 2024-11-08T18:41:42,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f2afebe1436a43d78e636a151b3eab27 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/f2afebe1436a43d78e636a151b3eab27 2024-11-08T18:41:42,720 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/recovered.edits/302.seqid, newMaxSeqId=302, maxSeqId=4 2024-11-08T18:41:42,721 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. 2024-11-08T18:41:42,721 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] regionserver.HRegion(1676): Region close journal for 2d8b2888d7998b4dc696cf695019721f: Waiting for close lock at 1731091300544Running coprocessor pre-close hooks at 1731091300544Disabling compacts and flushes for region at 1731091300544Disabling writes for close at 1731091300915 (+371 ms)Obtaining lock to block concurrent updates at 1731091300915Preparing flush snapshotting stores in 2d8b2888d7998b4dc696cf695019721f at 1731091300915Finished memstore snapshotting TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731091300915Flushing stores of TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f. at 1731091300916 (+1 ms)Flushing 2d8b2888d7998b4dc696cf695019721f/A: creating writer at 1731091300916Flushing mob file 2d8b2888d7998b4dc696cf695019721f/A: appending metadata at 1731091300921 (+5 ms)Flushing mob file 2d8b2888d7998b4dc696cf695019721f/A: closing flushed file at 1731091300921Flushing 2d8b2888d7998b4dc696cf695019721f/A: appending metadata at 1731091301334 (+413 ms)Flushing 2d8b2888d7998b4dc696cf695019721f/A: closing flushed file at 1731091301334Flushing 2d8b2888d7998b4dc696cf695019721f/B: creating writer at 1731091301778 (+444 ms)Flushing 2d8b2888d7998b4dc696cf695019721f/B: appending metadata at 1731091301784 (+6 ms)Flushing 2d8b2888d7998b4dc696cf695019721f/B: closing flushed file at 1731091301784Flushing 2d8b2888d7998b4dc696cf695019721f/C: creating writer at 1731091302197 (+413 ms)Flushing 2d8b2888d7998b4dc696cf695019721f/C: appending metadata at 1731091302201 (+4 ms)Flushing 2d8b2888d7998b4dc696cf695019721f/C: closing flushed file at 1731091302201Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50f4db5d: reopening flushed file at 1731091302615 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38f78bc: reopening flushed file at 1731091302621 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a5c88b9: reopening flushed file at 1731091302626 (+5 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 2d8b2888d7998b4dc696cf695019721f in 1717ms, sequenceid=299, compaction requested=false at 1731091302632 (+6 ms)Writing region close event to WAL at 1731091302717 (+85 ms)Running coprocessor post-close hooks at 1731091302721 (+4 ms)Closed at 1731091302721 2024-11-08T18:41:42,724 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=112}] handler.UnassignRegionHandler(157): Closed 2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:42,725 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=111 updating hbase:meta row=2d8b2888d7998b4dc696cf695019721f, regionState=CLOSED 2024-11-08T18:41:42,726 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=112, ppid=111, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:41:42,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-08T18:41:42,730 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseRegionProcedure 2d8b2888d7998b4dc696cf695019721f, server=b75fe890c219,44447,1731091183487 in 2.3380 sec 2024-11-08T18:41:42,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=111, resume processing ppid=110 2024-11-08T18:41:42,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, ppid=110, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2d8b2888d7998b4dc696cf695019721f, UNASSIGN in 2.3440 sec 2024-11-08T18:41:42,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-08T18:41:42,733 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.3480 sec 2024-11-08T18:41:42,734 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091302734"}]},"ts":"1731091302734"} 2024-11-08T18:41:42,735 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T18:41:42,735 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T18:41:42,736 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.3610 sec 2024-11-08T18:41:43,025 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T18:41:44,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-08T18:41:44,550 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:44,552 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T18:41:44,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:44,555 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=113, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:44,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T18:41:44,557 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=113, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:44,560 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,564 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/recovered.edits] 2024-11-08T18:41:44,569 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0d83c30a5adf40cfaad0209ce77527e0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/0d83c30a5adf40cfaad0209ce77527e0 2024-11-08T18:41:44,571 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f955cdee4c4c44f5b8c287f5b233fbfa to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/A/f955cdee4c4c44f5b8c287f5b233fbfa 2024-11-08T18:41:44,575 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/46150377e5ad40948153fa246fd9b35a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/46150377e5ad40948153fa246fd9b35a 2024-11-08T18:41:44,577 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/7174dc628f2e4aa1adfbfce4968b708a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/B/7174dc628f2e4aa1adfbfce4968b708a 2024-11-08T18:41:44,580 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/43514520294d4aab80d3385dff3f1436 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/43514520294d4aab80d3385dff3f1436 2024-11-08T18:41:44,581 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/c65bf870f765406a8787f1b8718530c2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/C/c65bf870f765406a8787f1b8718530c2 2024-11-08T18:41:44,584 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/recovered.edits/302.seqid to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f/recovered.edits/302.seqid 2024-11-08T18:41:44,585 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,585 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T18:41:44,585 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T18:41:44,586 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T18:41:44,590 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110800edfad603b14d19b8672357411c7e29_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110800edfad603b14d19b8672357411c7e29_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,592 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080bf8446e08e94102aaa003c5179b127c_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080bf8446e08e94102aaa003c5179b127c_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,594 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c5b3f20948a4857ae9a94834caca7ac_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411080c5b3f20948a4857ae9a94834caca7ac_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,596 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110812bdefe7e94445a6886676ea142b95e2_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110812bdefe7e94445a6886676ea142b95e2_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,598 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110818becba9b17c4c7fa3691f884b064940_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110818becba9b17c4c7fa3691f884b064940_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,600 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110828ab128524a94040bd07e0eaf2f6eb12_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110828ab128524a94040bd07e0eaf2f6eb12_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,601 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b9816ee100143148def5bf3b9327de1_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083b9816ee100143148def5bf3b9327de1_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,603 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083bf83033b8654e67aeacda12e4ba6628_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411083bf83033b8654e67aeacda12e4ba6628_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,604 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085390edf9b88646858500d4fa600c4d34_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085390edf9b88646858500d4fa600c4d34_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,605 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110853f7d1fff2594fcf89820f386ce06714_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110853f7d1fff2594fcf89820f386ce06714_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,606 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108576e28ec8a404460b3ba65aa810805f2_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108576e28ec8a404460b3ba65aa810805f2_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,606 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108763850b2da554e7fb574927b980bf9f8_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108763850b2da554e7fb574927b980bf9f8_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,607 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088410219160ac43a9bbc66212c7134354_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411088410219160ac43a9bbc66212c7134354_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,608 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b433dddb1647475394ccb0166645d6a3_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108b433dddb1647475394ccb0166645d6a3_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,609 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c2df614dd66b4a3c82e891809c1ec295_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c2df614dd66b4a3c82e891809c1ec295_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,610 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108dc28a8a92f464766a2e2371541ead234_2d8b2888d7998b4dc696cf695019721f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108dc28a8a92f464766a2e2371541ead234_2d8b2888d7998b4dc696cf695019721f 2024-11-08T18:41:44,610 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T18:41:44,612 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=113, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:44,614 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T18:41:44,615 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T18:41:44,616 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=113, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:44,616 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T18:41:44,617 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731091304616"}]},"ts":"9223372036854775807"} 2024-11-08T18:41:44,618 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T18:41:44,618 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2d8b2888d7998b4dc696cf695019721f, NAME => 'TestAcidGuarantees,,1731091274869.2d8b2888d7998b4dc696cf695019721f.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T18:41:44,618 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T18:41:44,619 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731091304618"}]},"ts":"9223372036854775807"} 2024-11-08T18:41:44,620 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T18:41:44,621 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=113, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:44,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 69 msec 2024-11-08T18:41:44,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-08T18:41:44,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T18:41:44,660 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:44,668 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 239) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=353 (was 407), ProcessCount=11 (was 11), AvailableMemoryMB=3728 (was 3769) 2024-11-08T18:41:44,675 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=353, ProcessCount=11, AvailableMemoryMB=3728 2024-11-08T18:41:44,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:41:44,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:41:44,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T18:41:44,678 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T18:41:44,678 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:44,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 114 2024-11-08T18:41:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-08T18:41:44,679 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T18:41:44,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742250_1426 (size=959) 2024-11-08T18:41:44,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-08T18:41:45,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-08T18:41:45,090 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 2024-11-08T18:41:45,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742251_1427 (size=53) 2024-11-08T18:41:45,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-08T18:41:45,500 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:41:45,500 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 59c8200f26f4f9ab5bce4184c7b0cb59, disabling compactions & flushes 2024-11-08T18:41:45,500 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:45,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:45,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. after waiting 0 ms 2024-11-08T18:41:45,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:45,501 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:45,501 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 59c8200f26f4f9ab5bce4184c7b0cb59: Waiting for close lock at 1731091305500Disabling compacts and flushes for region at 1731091305500Disabling writes for close at 1731091305501 (+1 ms)Writing region close event to WAL at 1731091305501Closed at 1731091305501 2024-11-08T18:41:45,504 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T18:41:45,504 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731091305504"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731091305504"}]},"ts":"1731091305504"} 2024-11-08T18:41:45,507 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T18:41:45,509 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T18:41:45,509 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091305509"}]},"ts":"1731091305509"} 2024-11-08T18:41:45,512 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T18:41:45,512 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, ASSIGN}] 2024-11-08T18:41:45,514 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, ASSIGN 2024-11-08T18:41:45,514 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, ASSIGN; state=OFFLINE, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=false 2024-11-08T18:41:45,666 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=59c8200f26f4f9ab5bce4184c7b0cb59, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:45,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, ASSIGN because future has completed 2024-11-08T18:41:45,671 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:41:45,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-08T18:41:45,833 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:45,833 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7752): Opening region: {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:41:45,834 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,834 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:41:45,834 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7794): checking encryption for 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,834 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(7797): checking classloading for 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,836 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,837 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59c8200f26f4f9ab5bce4184c7b0cb59 columnFamilyName A 2024-11-08T18:41:45,837 DEBUG [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:45,838 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] regionserver.HStore(327): Store=59c8200f26f4f9ab5bce4184c7b0cb59/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:45,838 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,839 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59c8200f26f4f9ab5bce4184c7b0cb59 columnFamilyName B 2024-11-08T18:41:45,839 DEBUG [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:45,839 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] regionserver.HStore(327): Store=59c8200f26f4f9ab5bce4184c7b0cb59/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:45,840 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,841 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 59c8200f26f4f9ab5bce4184c7b0cb59 columnFamilyName C 2024-11-08T18:41:45,841 DEBUG [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:41:45,841 INFO [StoreOpener-59c8200f26f4f9ab5bce4184c7b0cb59-1 {}] regionserver.HStore(327): Store=59c8200f26f4f9ab5bce4184c7b0cb59/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:41:45,841 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1038): replaying wal for 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,842 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,842 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,843 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1048): stopping wal replay for 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,843 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1060): Cleaning up temporary data for 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,844 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:41:45,845 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1093): writing seq id for 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,847 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T18:41:45,848 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1114): Opened 59c8200f26f4f9ab5bce4184c7b0cb59; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60369889, jitterRate=-0.10041855275630951}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:41:45,848 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:45,848 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegion(1006): Region open journal for 59c8200f26f4f9ab5bce4184c7b0cb59: Running coprocessor pre-open hook at 1731091305834Writing region info on filesystem at 1731091305834Initializing all the Stores at 1731091305835 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091305835Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091305836 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091305836Cleaning up temporary data from old regions at 1731091305843 (+7 ms)Running coprocessor post-open hooks at 1731091305848 (+5 ms)Region opened successfully at 1731091305848 2024-11-08T18:41:45,849 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., pid=116, masterSystemTime=1731091305825 2024-11-08T18:41:45,852 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:45,852 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=116}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:45,852 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=59c8200f26f4f9ab5bce4184c7b0cb59, regionState=OPEN, openSeqNum=2, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:41:45,854 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; OpenRegionProcedure 59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:41:45,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-08T18:41:45,858 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; OpenRegionProcedure 59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 in 185 msec 2024-11-08T18:41:45,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-08T18:41:45,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, ASSIGN in 346 msec 2024-11-08T18:41:45,862 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T18:41:45,862 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091305862"}]},"ts":"1731091305862"} 2024-11-08T18:41:45,864 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T18:41:45,865 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=114, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T18:41:45,867 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1880 sec 2024-11-08T18:41:46,159 WARN [regionserver/b75fe890c219:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 3, running: 1 2024-11-08T18:41:46,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-11-08T18:41:46,841 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:46,844 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be1708d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,844 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,844 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,846 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c244820, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,846 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,847 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb9e9b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,848 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,849 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,850 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,851 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T18:41:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is b75fe890c219,42293,1731091182659 2024-11-08T18:41:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1ed5dd0f 2024-11-08T18:41:46,852 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T18:41:46,853 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T18:41:46,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fe0696, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,855 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,855 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,855 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b61b744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,856 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,857 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43128, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32604cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,859 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,859 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,860 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,860 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ca301d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,861 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,862 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43140, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,862 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ddc811, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,863 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,863 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,863 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,864 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@773cbe41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,866 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2dcf2af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,868 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,868 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,868 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,869 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b6720e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,870 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43174, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cee7d1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,872 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,872 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,873 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3294e832, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,875 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,875 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@671fdd5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,876 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6bc5e64, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,876 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,876 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,877 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd929cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,877 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,878 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,878 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53209d70, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,880 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,880 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,881 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a78a31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,883 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,883 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13afb643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,884 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,884 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,885 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,885 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41f96978, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,886 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,886 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,887 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@196b314b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,887 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,887 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,888 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a55ac4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,888 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,889 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,889 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff530d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,890 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,890 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:41:46,890 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:41:46,891 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:41:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:41:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:41:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49e794e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:41:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:41:46,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:41:46,892 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:41:46,892 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56bc4e6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:41:46,893 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,893 DEBUG [Thread-1947 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,893 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,893 DEBUG [Thread-1939 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,893 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,893 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,893 DEBUG [Thread-1936 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,894 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,894 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,894 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,894 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,894 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,895 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,895 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,897 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33594, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,897 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,897 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:41:46,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,898 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,898 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:41:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:41:46,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,900 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,900 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,901 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:41:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2] 2024-11-08T18:41:46,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-08T18:41:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T18:41:46,934 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:46,936 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:46,937 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:46,937 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:46,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:41:46,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/de5d1ee5273247f886cd20d118c83796 is 50, key is test_row_0/A:col10/1731091306900/Put/seqid=0 2024-11-08T18:41:46,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742252_1428 (size=12001) 2024-11-08T18:41:46,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:46,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091366976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:46,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091366977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:46,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091366978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:46,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:46,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091366979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:46,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:46,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091366982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:46,988 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T18:41:47,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091367089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091367090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091367090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,095 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091367090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091367090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:47,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T18:41:47,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091367300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091367300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091367300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,305 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091367300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091367300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,306 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/de5d1ee5273247f886cd20d118c83796 2024-11-08T18:41:47,388 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/ba0da29381ab4bc09485e7977c66e642 is 50, key is test_row_0/B:col10/1731091306900/Put/seqid=0 2024-11-08T18:41:47,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742253_1429 (size=12001) 2024-11-08T18:41:47,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T18:41:47,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091367610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,612 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091367610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091367610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091367610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091367610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:47,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:47,700 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:47,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:47,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,791 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/ba0da29381ab4bc09485e7977c66e642 2024-11-08T18:41:47,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e7e5ba0fb1524d6483ec0f376f9c7ebd is 50, key is test_row_0/C:col10/1731091306900/Put/seqid=0 2024-11-08T18:41:47,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742254_1430 (size=12001) 2024-11-08T18:41:47,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:47,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:47,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:47,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:47,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:48,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:48,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:48,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:48,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:48,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:48,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:48,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T18:41:48,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091368120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091368120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091368120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:48,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091368120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:48,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091368120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:48,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:48,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:48,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:48,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:48,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:48,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:48,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:48,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e7e5ba0fb1524d6483ec0f376f9c7ebd 2024-11-08T18:41:48,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/de5d1ee5273247f886cd20d118c83796 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/de5d1ee5273247f886cd20d118c83796 2024-11-08T18:41:48,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/de5d1ee5273247f886cd20d118c83796, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T18:41:48,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/ba0da29381ab4bc09485e7977c66e642 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/ba0da29381ab4bc09485e7977c66e642 2024-11-08T18:41:48,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/ba0da29381ab4bc09485e7977c66e642, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T18:41:48,224 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e7e5ba0fb1524d6483ec0f376f9c7ebd as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e7e5ba0fb1524d6483ec0f376f9c7ebd 2024-11-08T18:41:48,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e7e5ba0fb1524d6483ec0f376f9c7ebd, entries=150, sequenceid=12, filesize=11.7 K 2024-11-08T18:41:48,229 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1292ms, sequenceid=12, compaction requested=false 2024-11-08T18:41:48,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:48,239 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T18:41:48,310 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-08T18:41:48,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:48,310 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-08T18:41:48,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/5fb5002cceea44ea92883c4cb3ffaf8d is 50, key is test_row_0/A:col10/1731091306975/Put/seqid=0 2024-11-08T18:41:48,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742255_1431 (size=12001) 2024-11-08T18:41:48,718 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/5fb5002cceea44ea92883c4cb3ffaf8d 2024-11-08T18:41:48,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c4429e70727c409ba09c09f7ea3bd3ba is 50, key is test_row_0/B:col10/1731091306975/Put/seqid=0 2024-11-08T18:41:48,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742256_1432 (size=12001) 2024-11-08T18:41:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T18:41:49,127 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c4429e70727c409ba09c09f7ea3bd3ba 2024-11-08T18:41:49,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/5d374bb78da44140990f2c0f476eccef is 50, key is test_row_0/C:col10/1731091306975/Put/seqid=0 2024-11-08T18:41:49,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:49,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:49,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742257_1433 (size=12001) 2024-11-08T18:41:49,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091369142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091369145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091369145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091369146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091369148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091369260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,265 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091369260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091369260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,266 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091369260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091369260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091369469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091369469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091369469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091369469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091369469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,471 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,543 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/5d374bb78da44140990f2c0f476eccef 2024-11-08T18:41:49,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/5fb5002cceea44ea92883c4cb3ffaf8d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/5fb5002cceea44ea92883c4cb3ffaf8d 2024-11-08T18:41:49,549 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/5fb5002cceea44ea92883c4cb3ffaf8d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T18:41:49,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c4429e70727c409ba09c09f7ea3bd3ba as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c4429e70727c409ba09c09f7ea3bd3ba 2024-11-08T18:41:49,552 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c4429e70727c409ba09c09f7ea3bd3ba, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T18:41:49,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/5d374bb78da44140990f2c0f476eccef as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5d374bb78da44140990f2c0f476eccef 2024-11-08T18:41:49,555 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5d374bb78da44140990f2c0f476eccef, entries=150, sequenceid=37, filesize=11.7 K 2024-11-08T18:41:49,555 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1245ms, sequenceid=37, compaction requested=false 2024-11-08T18:41:49,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:49,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:49,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-08T18:41:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-08T18:41:49,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-08T18:41:49,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6200 sec 2024-11-08T18:41:49,559 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.6600 sec 2024-11-08T18:41:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:49,782 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:49,782 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T18:41:49,797 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/2e81027587824d528a81db0265167477 is 50, key is test_row_0/A:col10/1731091309143/Put/seqid=0 2024-11-08T18:41:49,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742258_1434 (size=12001) 2024-11-08T18:41:49,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091369820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091369821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091369827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091369829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091369830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091369939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091369939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091369939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091369939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,945 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:49,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091369950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:49,954 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091370149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091370149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091370149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091370149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,165 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091370160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,166 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/2e81027587824d528a81db0265167477 2024-11-08T18:41:50,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/5305fa8a131d49839777d86ee23289f0 is 50, key is test_row_0/B:col10/1731091309143/Put/seqid=0 2024-11-08T18:41:50,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742259_1435 (size=12001) 2024-11-08T18:41:50,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091370459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091370459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091370459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,465 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,474 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091370469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,483 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091370479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,483 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/5305fa8a131d49839777d86ee23289f0 2024-11-08T18:41:50,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/41a8f40fee664a80805ab609b4ef9476 is 50, key is test_row_0/C:col10/1731091309143/Put/seqid=0 2024-11-08T18:41:50,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742260_1436 (size=12001) 2024-11-08T18:41:50,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091370969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091370969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091370969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,973 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,974 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091370979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:50,992 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:50,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091370990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:50,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:51,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/41a8f40fee664a80805ab609b4ef9476 2024-11-08T18:41:51,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/2e81027587824d528a81db0265167477 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/2e81027587824d528a81db0265167477 2024-11-08T18:41:51,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/2e81027587824d528a81db0265167477, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T18:41:51,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/5305fa8a131d49839777d86ee23289f0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/5305fa8a131d49839777d86ee23289f0 2024-11-08T18:41:51,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/5305fa8a131d49839777d86ee23289f0, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T18:41:51,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/41a8f40fee664a80805ab609b4ef9476 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/41a8f40fee664a80805ab609b4ef9476 2024-11-08T18:41:51,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/41a8f40fee664a80805ab609b4ef9476, entries=150, sequenceid=50, filesize=11.7 K 2024-11-08T18:41:51,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1248ms, sequenceid=50, compaction requested=true 2024-11-08T18:41:51,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:51,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:51,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:51,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:51,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:51,030 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:51,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:51,030 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:51,030 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:51,031 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:51,031 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:51,031 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/A is initiating minor compaction (all files) 2024-11-08T18:41:51,031 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/B is initiating minor compaction (all files) 2024-11-08T18:41:51,031 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/B in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:51,031 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/A in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:51,031 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/de5d1ee5273247f886cd20d118c83796, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/5fb5002cceea44ea92883c4cb3ffaf8d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/2e81027587824d528a81db0265167477] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.2 K 2024-11-08T18:41:51,031 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/ba0da29381ab4bc09485e7977c66e642, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c4429e70727c409ba09c09f7ea3bd3ba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/5305fa8a131d49839777d86ee23289f0] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.2 K 2024-11-08T18:41:51,031 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting de5d1ee5273247f886cd20d118c83796, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731091306900 2024-11-08T18:41:51,031 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting ba0da29381ab4bc09485e7977c66e642, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731091306900 2024-11-08T18:41:51,032 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting c4429e70727c409ba09c09f7ea3bd3ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731091306975 2024-11-08T18:41:51,032 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb5002cceea44ea92883c4cb3ffaf8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731091306975 2024-11-08T18:41:51,032 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e81027587824d528a81db0265167477, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731091309143 2024-11-08T18:41:51,032 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5305fa8a131d49839777d86ee23289f0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731091309143 2024-11-08T18:41:51,036 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#B#compaction#363 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:51,037 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/3d5ce44cc962452cbaaa971b19879066 is 50, key is test_row_0/B:col10/1731091309143/Put/seqid=0 2024-11-08T18:41:51,042 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#A#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:51,043 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f8892ace61fd4ab1ad04819860aee9fd is 50, key is test_row_0/A:col10/1731091309143/Put/seqid=0 2024-11-08T18:41:51,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742261_1437 (size=12104) 2024-11-08T18:41:51,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742262_1438 (size=12104) 2024-11-08T18:41:51,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-08T18:41:51,100 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:51,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:51,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-08T18:41:51,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T18:41:51,102 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:51,103 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:51,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:51,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T18:41:51,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-08T18:41:51,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:51,255 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:41:51,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/b4c131dffc964011a119c9a4cf68944c is 50, key is test_row_0/A:col10/1731091309827/Put/seqid=0 2024-11-08T18:41:51,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742263_1439 (size=12001) 2024-11-08T18:41:51,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T18:41:51,447 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/3d5ce44cc962452cbaaa971b19879066 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/3d5ce44cc962452cbaaa971b19879066 2024-11-08T18:41:51,449 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f8892ace61fd4ab1ad04819860aee9fd as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f8892ace61fd4ab1ad04819860aee9fd 2024-11-08T18:41:51,452 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/B of 59c8200f26f4f9ab5bce4184c7b0cb59 into 3d5ce44cc962452cbaaa971b19879066(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:51,452 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:51,452 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/B, priority=13, startTime=1731091311030; duration=0sec 2024-11-08T18:41:51,452 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:51,452 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:B 2024-11-08T18:41:51,452 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:51,453 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/A of 59c8200f26f4f9ab5bce4184c7b0cb59 into f8892ace61fd4ab1ad04819860aee9fd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:51,453 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:51,453 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/A, priority=13, startTime=1731091311030; duration=0sec 2024-11-08T18:41:51,453 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:51,453 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:51,453 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/C is initiating minor compaction (all files) 2024-11-08T18:41:51,453 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:A 2024-11-08T18:41:51,453 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/C in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:51,453 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e7e5ba0fb1524d6483ec0f376f9c7ebd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5d374bb78da44140990f2c0f476eccef, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/41a8f40fee664a80805ab609b4ef9476] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.2 K 2024-11-08T18:41:51,453 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting e7e5ba0fb1524d6483ec0f376f9c7ebd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731091306900 2024-11-08T18:41:51,454 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d374bb78da44140990f2c0f476eccef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731091306975 2024-11-08T18:41:51,454 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a8f40fee664a80805ab609b4ef9476, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731091309143 2024-11-08T18:41:51,459 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#C#compaction#366 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:51,460 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/a60c91138e1c4460a8bde6c185edadb4 is 50, key is test_row_0/C:col10/1731091309143/Put/seqid=0 2024-11-08T18:41:51,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742264_1440 (size=12104) 2024-11-08T18:41:51,663 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/b4c131dffc964011a119c9a4cf68944c 2024-11-08T18:41:51,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1e2529f905514ec5926234db0c43bfbb is 50, key is test_row_0/B:col10/1731091309827/Put/seqid=0 2024-11-08T18:41:51,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742265_1441 (size=12001) 2024-11-08T18:41:51,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T18:41:51,866 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/a60c91138e1c4460a8bde6c185edadb4 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a60c91138e1c4460a8bde6c185edadb4 2024-11-08T18:41:51,869 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/C of 59c8200f26f4f9ab5bce4184c7b0cb59 into a60c91138e1c4460a8bde6c185edadb4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:51,869 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:51,869 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/C, priority=13, startTime=1731091311030; duration=0sec 2024-11-08T18:41:51,869 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:51,869 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:C 2024-11-08T18:41:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:51,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:52,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091372002, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091372003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091372004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091372004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091372007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,076 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1e2529f905514ec5926234db0c43bfbb 2024-11-08T18:41:52,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/ffd30b29f81a4249a98d086d9abe81fa is 50, key is test_row_0/C:col10/1731091309827/Put/seqid=0 2024-11-08T18:41:52,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742266_1442 (size=12001) 2024-11-08T18:41:52,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091372110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091372119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091372119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091372119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091372119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,126 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T18:41:52,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091372319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,327 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091372329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091372329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091372329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091372329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,340 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,491 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=73 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/ffd30b29f81a4249a98d086d9abe81fa 2024-11-08T18:41:52,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/b4c131dffc964011a119c9a4cf68944c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/b4c131dffc964011a119c9a4cf68944c 2024-11-08T18:41:52,498 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/b4c131dffc964011a119c9a4cf68944c, entries=150, sequenceid=73, filesize=11.7 K 2024-11-08T18:41:52,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1e2529f905514ec5926234db0c43bfbb as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1e2529f905514ec5926234db0c43bfbb 2024-11-08T18:41:52,501 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1e2529f905514ec5926234db0c43bfbb, entries=150, sequenceid=73, filesize=11.7 K 2024-11-08T18:41:52,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/ffd30b29f81a4249a98d086d9abe81fa as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/ffd30b29f81a4249a98d086d9abe81fa 2024-11-08T18:41:52,505 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/ffd30b29f81a4249a98d086d9abe81fa, entries=150, sequenceid=73, filesize=11.7 K 2024-11-08T18:41:52,506 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1251ms, sequenceid=73, compaction requested=false 2024-11-08T18:41:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:52,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-08T18:41:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-08T18:41:52,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-08T18:41:52,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4040 sec 2024-11-08T18:41:52,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.4090 sec 2024-11-08T18:41:52,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:52,632 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:52,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:41:52,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f93315ed71534935a7f6bbe37c2cbaba is 50, key is test_row_0/A:col10/1731091311995/Put/seqid=0 2024-11-08T18:41:52,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742267_1443 (size=14341) 2024-11-08T18:41:52,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091372674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,685 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091372675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091372684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091372685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091372686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091372789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091372789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091372789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,797 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091372800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:52,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091372800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:52,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,005 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091372999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091372999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091372999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,024 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-08T18:41:53,024 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-08T18:41:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091373020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091373020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,054 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f93315ed71534935a7f6bbe37c2cbaba 2024-11-08T18:41:53,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c25eee6303044a95b1f6bd3896730d67 is 50, key is test_row_0/B:col10/1731091311995/Put/seqid=0 2024-11-08T18:41:53,062 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742268_1444 (size=12001) 2024-11-08T18:41:53,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-08T18:41:53,250 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:53,250 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-08T18:41:53,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T18:41:53,251 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:53,252 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:53,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:53,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091373309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091373309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091373319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091373329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091373330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T18:41:53,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T18:41:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c25eee6303044a95b1f6bd3896730d67 2024-11-08T18:41:53,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/04f1e686465045678bbaa5932ef1203f is 50, key is test_row_0/C:col10/1731091311995/Put/seqid=0 2024-11-08T18:41:53,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742269_1445 (size=12001) 2024-11-08T18:41:53,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T18:41:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:53,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T18:41:53,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T18:41:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091373819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091373820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091373829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,833 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091373839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,843 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:53,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091373849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:53,857 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:53,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T18:41:53,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,862 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:53,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/04f1e686465045678bbaa5932ef1203f 2024-11-08T18:41:53,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f93315ed71534935a7f6bbe37c2cbaba as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f93315ed71534935a7f6bbe37c2cbaba 2024-11-08T18:41:53,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f93315ed71534935a7f6bbe37c2cbaba, entries=200, sequenceid=90, filesize=14.0 K 2024-11-08T18:41:53,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c25eee6303044a95b1f6bd3896730d67 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c25eee6303044a95b1f6bd3896730d67 2024-11-08T18:41:53,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T18:41:53,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c25eee6303044a95b1f6bd3896730d67, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T18:41:53,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/04f1e686465045678bbaa5932ef1203f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/04f1e686465045678bbaa5932ef1203f 2024-11-08T18:41:53,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/04f1e686465045678bbaa5932ef1203f, entries=150, sequenceid=90, filesize=11.7 K 2024-11-08T18:41:53,885 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1253ms, sequenceid=90, compaction requested=true 2024-11-08T18:41:53,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:53,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:53,886 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:53,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:53,886 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:53,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:53,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:53,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:53,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:53,886 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:53,886 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:53,886 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/B is initiating minor compaction (all files) 2024-11-08T18:41:53,886 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/A is initiating minor compaction (all files) 2024-11-08T18:41:53,886 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/A in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,886 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/B in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,886 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/3d5ce44cc962452cbaaa971b19879066, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1e2529f905514ec5926234db0c43bfbb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c25eee6303044a95b1f6bd3896730d67] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.3 K 2024-11-08T18:41:53,886 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f8892ace61fd4ab1ad04819860aee9fd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/b4c131dffc964011a119c9a4cf68944c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f93315ed71534935a7f6bbe37c2cbaba] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=37.5 K 2024-11-08T18:41:53,887 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8892ace61fd4ab1ad04819860aee9fd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731091309143 2024-11-08T18:41:53,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d5ce44cc962452cbaaa971b19879066, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731091309143 2024-11-08T18:41:53,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e2529f905514ec5926234db0c43bfbb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731091309820 2024-11-08T18:41:53,887 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4c131dffc964011a119c9a4cf68944c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731091309820 2024-11-08T18:41:53,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting c25eee6303044a95b1f6bd3896730d67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091311995 2024-11-08T18:41:53,887 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f93315ed71534935a7f6bbe37c2cbaba, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091311995 2024-11-08T18:41:53,892 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#A#compaction#373 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:53,892 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#B#compaction#372 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:53,892 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/2649588d531047fdbf2b545c574dbd5d is 50, key is test_row_0/B:col10/1731091311995/Put/seqid=0 2024-11-08T18:41:53,892 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/654c628d55814bc4a60d20b522582ecc is 50, key is test_row_0/A:col10/1731091311995/Put/seqid=0 2024-11-08T18:41:53,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742271_1447 (size=12207) 2024-11-08T18:41:53,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742270_1446 (size=12207) 2024-11-08T18:41:53,906 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/654c628d55814bc4a60d20b522582ecc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/654c628d55814bc4a60d20b522582ecc 2024-11-08T18:41:53,909 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/A of 59c8200f26f4f9ab5bce4184c7b0cb59 into 654c628d55814bc4a60d20b522582ecc(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:53,909 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:53,909 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/A, priority=13, startTime=1731091313886; duration=0sec 2024-11-08T18:41:53,909 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:53,909 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:A 2024-11-08T18:41:53,909 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:53,910 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:53,910 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/C is initiating minor compaction (all files) 2024-11-08T18:41:53,910 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/C in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:53,910 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a60c91138e1c4460a8bde6c185edadb4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/ffd30b29f81a4249a98d086d9abe81fa, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/04f1e686465045678bbaa5932ef1203f] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.3 K 2024-11-08T18:41:53,910 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a60c91138e1c4460a8bde6c185edadb4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731091309143 2024-11-08T18:41:53,910 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffd30b29f81a4249a98d086d9abe81fa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=73, earliestPutTs=1731091309820 2024-11-08T18:41:53,912 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f1e686465045678bbaa5932ef1203f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091311995 2024-11-08T18:41:53,917 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#C#compaction#374 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:53,917 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/948f8a36540f4b0ba3e72382bac950ac is 50, key is test_row_0/C:col10/1731091311995/Put/seqid=0 2024-11-08T18:41:53,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742272_1448 (size=12207) 2024-11-08T18:41:54,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-08T18:41:54,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:54,014 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:41:54,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/cb882d763bc649dc99451f814adf2e0b is 50, key is test_row_0/A:col10/1731091312684/Put/seqid=0 2024-11-08T18:41:54,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742273_1449 (size=12001) 2024-11-08T18:41:54,307 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/2649588d531047fdbf2b545c574dbd5d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/2649588d531047fdbf2b545c574dbd5d 2024-11-08T18:41:54,310 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/B of 59c8200f26f4f9ab5bce4184c7b0cb59 into 2649588d531047fdbf2b545c574dbd5d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:54,310 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:54,310 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/B, priority=13, startTime=1731091313886; duration=0sec 2024-11-08T18:41:54,311 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:54,311 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:B 2024-11-08T18:41:54,324 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/948f8a36540f4b0ba3e72382bac950ac as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/948f8a36540f4b0ba3e72382bac950ac 2024-11-08T18:41:54,328 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/C of 59c8200f26f4f9ab5bce4184c7b0cb59 into 948f8a36540f4b0ba3e72382bac950ac(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:54,328 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:54,328 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/C, priority=13, startTime=1731091313886; duration=0sec 2024-11-08T18:41:54,328 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:54,328 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:C 2024-11-08T18:41:54,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T18:41:54,421 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/cb882d763bc649dc99451f814adf2e0b 2024-11-08T18:41:54,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/9b1a098f615f41eb89c7fee7860ac26e is 50, key is test_row_0/B:col10/1731091312684/Put/seqid=0 2024-11-08T18:41:54,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742274_1450 (size=12001) 2024-11-08T18:41:54,831 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/9b1a098f615f41eb89c7fee7860ac26e 2024-11-08T18:41:54,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/dbed55b44f2f4d8083322528bc96d372 is 50, key is test_row_0/C:col10/1731091312684/Put/seqid=0 2024-11-08T18:41:54,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742275_1451 (size=12001) 2024-11-08T18:41:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:54,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:54,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091374854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091374856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:54,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091374857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:54,865 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091374857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:54,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091374870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,874 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091374969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091374970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091374970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091374970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,973 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:54,974 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091375179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091375179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091375179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091375180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,240 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/dbed55b44f2f4d8083322528bc96d372 2024-11-08T18:41:55,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/cb882d763bc649dc99451f814adf2e0b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/cb882d763bc649dc99451f814adf2e0b 2024-11-08T18:41:55,246 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/cb882d763bc649dc99451f814adf2e0b, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T18:41:55,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/9b1a098f615f41eb89c7fee7860ac26e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/9b1a098f615f41eb89c7fee7860ac26e 2024-11-08T18:41:55,250 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/9b1a098f615f41eb89c7fee7860ac26e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T18:41:55,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/dbed55b44f2f4d8083322528bc96d372 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/dbed55b44f2f4d8083322528bc96d372 2024-11-08T18:41:55,253 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/dbed55b44f2f4d8083322528bc96d372, entries=150, sequenceid=114, filesize=11.7 K 2024-11-08T18:41:55,254 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1240ms, sequenceid=114, compaction requested=false 2024-11-08T18:41:55,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:55,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:55,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-08T18:41:55,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-08T18:41:55,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-08T18:41:55,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0030 sec 2024-11-08T18:41:55,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.0070 sec 2024-11-08T18:41:55,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-08T18:41:55,410 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:55,410 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-08T18:41:55,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T18:41:55,411 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:55,412 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:55,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:55,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:55,493 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:55,493 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:41:55,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/09b6c893b73a461cb7fe5da4a55a8623 is 50, key is test_row_0/A:col10/1731091314854/Put/seqid=0 2024-11-08T18:41:55,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742276_1452 (size=16781) 2024-11-08T18:41:55,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T18:41:55,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091375528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091375529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,539 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091375530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091375531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,540 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:55,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091375639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091375649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091375649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091375649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:55,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:55,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:55,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:55,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T18:41:55,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091375849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,856 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091375859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091375859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,866 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091375859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:55,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:55,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:55,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:55,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/09b6c893b73a461cb7fe5da4a55a8623 2024-11-08T18:41:55,909 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/8603e7a9af5f46cbab5644d4cd84e94a is 50, key is test_row_0/B:col10/1731091314854/Put/seqid=0 2024-11-08T18:41:55,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742277_1453 (size=12101) 2024-11-08T18:41:56,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:56,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T18:41:56,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091376159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:56,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:56,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:56,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091376170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091376170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,178 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:56,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091376179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:56,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/8603e7a9af5f46cbab5644d4cd84e94a 2024-11-08T18:41:56,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/8b1d481f858f4a549478a8c9954fa9bc is 50, key is test_row_0/C:col10/1731091314854/Put/seqid=0 2024-11-08T18:41:56,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742278_1454 (size=12101) 2024-11-08T18:41:56,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T18:41:56,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:56,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:56,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091376680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:56,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091376689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091376689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091376689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,698 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:56,726 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=131 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/8b1d481f858f4a549478a8c9954fa9bc 2024-11-08T18:41:56,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/09b6c893b73a461cb7fe5da4a55a8623 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/09b6c893b73a461cb7fe5da4a55a8623 2024-11-08T18:41:56,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/09b6c893b73a461cb7fe5da4a55a8623, entries=250, sequenceid=131, filesize=16.4 K 2024-11-08T18:41:56,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/8603e7a9af5f46cbab5644d4cd84e94a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8603e7a9af5f46cbab5644d4cd84e94a 2024-11-08T18:41:56,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8603e7a9af5f46cbab5644d4cd84e94a, entries=150, sequenceid=131, filesize=11.8 K 2024-11-08T18:41:56,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/8b1d481f858f4a549478a8c9954fa9bc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/8b1d481f858f4a549478a8c9954fa9bc 2024-11-08T18:41:56,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/8b1d481f858f4a549478a8c9954fa9bc, entries=150, sequenceid=131, filesize=11.8 K 2024-11-08T18:41:56,740 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1247ms, sequenceid=131, compaction requested=true 2024-11-08T18:41:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:56,740 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:56,740 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:56,740 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:56,741 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40989 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:56,741 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:56,741 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/A is initiating minor compaction (all files) 2024-11-08T18:41:56,741 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/B is initiating minor compaction (all files) 2024-11-08T18:41:56,741 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/A in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,741 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/B in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,741 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/654c628d55814bc4a60d20b522582ecc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/cb882d763bc649dc99451f814adf2e0b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/09b6c893b73a461cb7fe5da4a55a8623] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=40.0 K 2024-11-08T18:41:56,741 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/2649588d531047fdbf2b545c574dbd5d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/9b1a098f615f41eb89c7fee7860ac26e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8603e7a9af5f46cbab5644d4cd84e94a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.5 K 2024-11-08T18:41:56,741 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654c628d55814bc4a60d20b522582ecc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091311995 2024-11-08T18:41:56,741 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 2649588d531047fdbf2b545c574dbd5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091311995 2024-11-08T18:41:56,742 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb882d763bc649dc99451f814adf2e0b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731091312669 2024-11-08T18:41:56,742 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b1a098f615f41eb89c7fee7860ac26e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731091312669 2024-11-08T18:41:56,742 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8603e7a9af5f46cbab5644d4cd84e94a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731091314854 2024-11-08T18:41:56,742 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09b6c893b73a461cb7fe5da4a55a8623, keycount=250, bloomtype=ROW, size=16.4 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731091314854 2024-11-08T18:41:56,754 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#A#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:56,754 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/517a2fe9b4004d1c994178d06862afb6 is 50, key is test_row_0/A:col10/1731091314854/Put/seqid=0 2024-11-08T18:41:56,756 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#B#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:56,757 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/a1f23c9c70eb4802ae26bbdf1b36a517 is 50, key is test_row_0/B:col10/1731091314854/Put/seqid=0 2024-11-08T18:41:56,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742279_1455 (size=12409) 2024-11-08T18:41:56,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742280_1456 (size=12409) 2024-11-08T18:41:56,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-08T18:41:56,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:56,786 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T18:41:56,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c22d1aec4e42403a9a15759d1115d26e is 50, key is test_row_0/A:col10/1731091315530/Put/seqid=0 2024-11-08T18:41:56,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742281_1457 (size=12151) 2024-11-08T18:41:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:56,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:56,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:56,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091376962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:56,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:57,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:57,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091377069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,075 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:57,194 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c22d1aec4e42403a9a15759d1115d26e 2024-11-08T18:41:57,200 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/a1f23c9c70eb4802ae26bbdf1b36a517 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a1f23c9c70eb4802ae26bbdf1b36a517 2024-11-08T18:41:57,200 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/517a2fe9b4004d1c994178d06862afb6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/517a2fe9b4004d1c994178d06862afb6 2024-11-08T18:41:57,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/e06c3bfe5f6f4723a47f5871a7d7d457 is 50, key is test_row_0/B:col10/1731091315530/Put/seqid=0 2024-11-08T18:41:57,204 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/B of 59c8200f26f4f9ab5bce4184c7b0cb59 into a1f23c9c70eb4802ae26bbdf1b36a517(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:57,204 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/A of 59c8200f26f4f9ab5bce4184c7b0cb59 into 517a2fe9b4004d1c994178d06862afb6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:57,204 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:57,204 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:57,204 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/A, priority=13, startTime=1731091316740; duration=0sec 2024-11-08T18:41:57,204 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/B, priority=13, startTime=1731091316740; duration=0sec 2024-11-08T18:41:57,204 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:57,204 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:57,204 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:A 2024-11-08T18:41:57,204 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:B 2024-11-08T18:41:57,204 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:57,205 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:57,205 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/C is initiating minor compaction (all files) 2024-11-08T18:41:57,205 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/C in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:57,205 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/948f8a36540f4b0ba3e72382bac950ac, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/dbed55b44f2f4d8083322528bc96d372, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/8b1d481f858f4a549478a8c9954fa9bc] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.5 K 2024-11-08T18:41:57,206 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 948f8a36540f4b0ba3e72382bac950ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1731091311995 2024-11-08T18:41:57,206 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting dbed55b44f2f4d8083322528bc96d372, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1731091312669 2024-11-08T18:41:57,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742282_1458 (size=12151) 2024-11-08T18:41:57,206 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b1d481f858f4a549478a8c9954fa9bc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731091314854 2024-11-08T18:41:57,211 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#C#compaction#385 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:57,212 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/17afde9f67d74951ac3964b7aa125c9b is 50, key is test_row_0/C:col10/1731091314854/Put/seqid=0 2024-11-08T18:41:57,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742283_1459 (size=12409) 2024-11-08T18:41:57,217 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/17afde9f67d74951ac3964b7aa125c9b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/17afde9f67d74951ac3964b7aa125c9b 2024-11-08T18:41:57,221 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/C of 59c8200f26f4f9ab5bce4184c7b0cb59 into 17afde9f67d74951ac3964b7aa125c9b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:57,222 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:57,222 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/C, priority=13, startTime=1731091316740; duration=0sec 2024-11-08T18:41:57,222 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:57,222 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:C 2024-11-08T18:41:57,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091377280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,285 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:57,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T18:41:57,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:57,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091377589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:57,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:57,607 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/e06c3bfe5f6f4723a47f5871a7d7d457 2024-11-08T18:41:57,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/9bd95c72d84a4c41be3b389dd33e8e6b is 50, key is test_row_0/C:col10/1731091315530/Put/seqid=0 2024-11-08T18:41:57,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742284_1460 (size=12151) 2024-11-08T18:41:57,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:57,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091377699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:57,703 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2175 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:57,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091377709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:57,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:57,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091377709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:57,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091377709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:57,713 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,713 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:57,714 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2184 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:57,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:57,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:58,016 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/9bd95c72d84a4c41be3b389dd33e8e6b 2024-11-08T18:41:58,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c22d1aec4e42403a9a15759d1115d26e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c22d1aec4e42403a9a15759d1115d26e 2024-11-08T18:41:58,022 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c22d1aec4e42403a9a15759d1115d26e, entries=150, sequenceid=152, filesize=11.9 K 2024-11-08T18:41:58,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/e06c3bfe5f6f4723a47f5871a7d7d457 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e06c3bfe5f6f4723a47f5871a7d7d457 2024-11-08T18:41:58,025 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e06c3bfe5f6f4723a47f5871a7d7d457, entries=150, sequenceid=152, filesize=11.9 K 2024-11-08T18:41:58,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/9bd95c72d84a4c41be3b389dd33e8e6b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/9bd95c72d84a4c41be3b389dd33e8e6b 2024-11-08T18:41:58,029 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/9bd95c72d84a4c41be3b389dd33e8e6b, entries=150, sequenceid=152, filesize=11.9 K 2024-11-08T18:41:58,030 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1244ms, sequenceid=152, compaction requested=false 2024-11-08T18:41:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:58,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-08T18:41:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-08T18:41:58,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-08T18:41:58,033 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6190 sec 2024-11-08T18:41:58,034 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 2.6230 sec 2024-11-08T18:41:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:58,102 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:58,102 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-08T18:41:58,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/9d0c01c6161f41fc92ed8160dbcb03c5 is 50, key is test_row_0/A:col10/1731091316934/Put/seqid=0 2024-11-08T18:41:58,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742285_1461 (size=14541) 2024-11-08T18:41:58,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:58,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091378212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:58,218 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:58,333 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091378329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:58,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:58,509 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/9d0c01c6161f41fc92ed8160dbcb03c5 2024-11-08T18:41:58,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd is 50, key is test_row_0/B:col10/1731091316934/Put/seqid=0 2024-11-08T18:41:58,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742286_1462 (size=12151) 2024-11-08T18:41:58,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:58,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091378539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:58,542 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:58,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091378849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:58,855 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:58,856 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:58,919 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd 2024-11-08T18:41:58,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/5ead5eabc4b1459fa7a0312428c64975 is 50, key is test_row_0/C:col10/1731091316934/Put/seqid=0 2024-11-08T18:41:58,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742287_1463 (size=12151) 2024-11-08T18:41:59,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/5ead5eabc4b1459fa7a0312428c64975 2024-11-08T18:41:59,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/9d0c01c6161f41fc92ed8160dbcb03c5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9d0c01c6161f41fc92ed8160dbcb03c5 2024-11-08T18:41:59,333 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9d0c01c6161f41fc92ed8160dbcb03c5, entries=200, sequenceid=171, filesize=14.2 K 2024-11-08T18:41:59,334 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd 2024-11-08T18:41:59,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T18:41:59,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/5ead5eabc4b1459fa7a0312428c64975 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5ead5eabc4b1459fa7a0312428c64975 2024-11-08T18:41:59,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5ead5eabc4b1459fa7a0312428c64975, entries=150, sequenceid=171, filesize=11.9 K 2024-11-08T18:41:59,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1238ms, sequenceid=171, compaction requested=true 2024-11-08T18:41:59,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:59,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:41:59,340 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:59,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:59,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:41:59,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:59,340 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:59,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:41:59,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:59,341 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39101 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:59,341 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/A is initiating minor compaction (all files) 2024-11-08T18:41:59,341 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:59,341 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/A in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:59,341 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/B is initiating minor compaction (all files) 2024-11-08T18:41:59,341 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/B in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:59,341 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/517a2fe9b4004d1c994178d06862afb6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c22d1aec4e42403a9a15759d1115d26e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9d0c01c6161f41fc92ed8160dbcb03c5] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=38.2 K 2024-11-08T18:41:59,341 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a1f23c9c70eb4802ae26bbdf1b36a517, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e06c3bfe5f6f4723a47f5871a7d7d457, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.9 K 2024-11-08T18:41:59,341 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 517a2fe9b4004d1c994178d06862afb6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731091314854 2024-11-08T18:41:59,341 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f23c9c70eb4802ae26bbdf1b36a517, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731091314854 2024-11-08T18:41:59,342 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c22d1aec4e42403a9a15759d1115d26e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731091315507 2024-11-08T18:41:59,342 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting e06c3bfe5f6f4723a47f5871a7d7d457, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731091315507 2024-11-08T18:41:59,342 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d0c01c6161f41fc92ed8160dbcb03c5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091316934 2024-11-08T18:41:59,342 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 143e2ddaf03d4bfe8b3fa603c8a7c2fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091316934 2024-11-08T18:41:59,347 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#B#compaction#391 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:59,347 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:59,347 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/a554df30737d424e9417777477534c26 is 50, key is test_row_0/B:col10/1731091316934/Put/seqid=0 2024-11-08T18:41:59,347 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f3d5243d11fe4a83836e5075d46d2b2f is 50, key is test_row_0/A:col10/1731091316934/Put/seqid=0 2024-11-08T18:41:59,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742289_1465 (size=12561) 2024-11-08T18:41:59,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742288_1464 (size=12561) 2024-11-08T18:41:59,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:41:59,375 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:41:59,375 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T18:41:59,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f084ef0eb815452b9ecc9db270e17f0f is 50, key is test_row_0/A:col10/1731091319374/Put/seqid=0 2024-11-08T18:41:59,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742290_1466 (size=14541) 2024-11-08T18:41:59,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:59,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091379449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:59,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:59,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091379560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:59,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:59,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-08T18:41:59,579 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:41:59,579 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:41:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-08T18:41:59,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T18:41:59,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:41:59,581 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:41:59,582 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:41:59,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T18:41:59,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:59,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091379719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:59,727 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,728 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:59,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T18:41:59,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:59,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:59,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:59,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:59,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:59,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:59,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:59,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091379729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:59,739 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,739 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091379740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:59,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:59,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091379740, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:59,746 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:59,746 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:41:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,747 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:59,754 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f3d5243d11fe4a83836e5075d46d2b2f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f3d5243d11fe4a83836e5075d46d2b2f 2024-11-08T18:41:59,754 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/a554df30737d424e9417777477534c26 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a554df30737d424e9417777477534c26 2024-11-08T18:41:59,757 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/A of 59c8200f26f4f9ab5bce4184c7b0cb59 into f3d5243d11fe4a83836e5075d46d2b2f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:59,757 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:59,757 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/A, priority=13, startTime=1731091319340; duration=0sec 2024-11-08T18:41:59,757 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:41:59,757 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:A 2024-11-08T18:41:59,757 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:41:59,757 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/B of 59c8200f26f4f9ab5bce4184c7b0cb59 into a554df30737d424e9417777477534c26(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:41:59,757 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:41:59,757 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/B, priority=13, startTime=1731091319340; duration=0sec 2024-11-08T18:41:59,757 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:41:59,758 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:B 2024-11-08T18:41:59,758 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:41:59,758 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/C is initiating minor compaction (all files) 2024-11-08T18:41:59,758 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/C in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:59,758 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/17afde9f67d74951ac3964b7aa125c9b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/9bd95c72d84a4c41be3b389dd33e8e6b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5ead5eabc4b1459fa7a0312428c64975] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=35.9 K 2024-11-08T18:41:59,759 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17afde9f67d74951ac3964b7aa125c9b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=131, earliestPutTs=1731091314854 2024-11-08T18:41:59,759 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bd95c72d84a4c41be3b389dd33e8e6b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731091315507 2024-11-08T18:41:59,759 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ead5eabc4b1459fa7a0312428c64975, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091316934 2024-11-08T18:41:59,764 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#C#compaction#393 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:41:59,765 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/5aaf5a17fe934733ba4e42ed105bb5a5 is 50, key is test_row_0/C:col10/1731091316934/Put/seqid=0 2024-11-08T18:41:59,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742291_1467 (size=12561) 2024-11-08T18:41:59,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:41:59,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091379769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:41:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:41:59,775 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:41:59,782 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f084ef0eb815452b9ecc9db270e17f0f 2024-11-08T18:41:59,787 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1327d49ffc3d4940a574695b745aba82 is 50, key is test_row_0/B:col10/1731091319374/Put/seqid=0 2024-11-08T18:41:59,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742292_1468 (size=12151) 2024-11-08T18:41:59,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T18:41:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:41:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:41:59,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:59,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:59,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:41:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T18:42:00,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T18:42:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:00,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:00,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:00,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:00,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:00,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:00,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:00,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091380090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:00,094 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:00,171 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/5aaf5a17fe934733ba4e42ed105bb5a5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5aaf5a17fe934733ba4e42ed105bb5a5 2024-11-08T18:42:00,175 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/C of 59c8200f26f4f9ab5bce4184c7b0cb59 into 5aaf5a17fe934733ba4e42ed105bb5a5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:00,175 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:00,175 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/C, priority=13, startTime=1731091319340; duration=0sec 2024-11-08T18:42:00,175 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:00,175 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:C 2024-11-08T18:42:00,190 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1327d49ffc3d4940a574695b745aba82 2024-11-08T18:42:00,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T18:42:00,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:00,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:00,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:00,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:00,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:00,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:00,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/f07882e99b9c4aea956bd7fa97c7655d is 50, key is test_row_0/C:col10/1731091319374/Put/seqid=0 2024-11-08T18:42:00,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T18:42:00,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742293_1469 (size=12151) 2024-11-08T18:42:00,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/f07882e99b9c4aea956bd7fa97c7655d 2024-11-08T18:42:00,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/f084ef0eb815452b9ecc9db270e17f0f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f084ef0eb815452b9ecc9db270e17f0f 2024-11-08T18:42:00,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f084ef0eb815452b9ecc9db270e17f0f, entries=200, sequenceid=192, filesize=14.2 K 2024-11-08T18:42:00,218 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1327d49ffc3d4940a574695b745aba82 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1327d49ffc3d4940a574695b745aba82 2024-11-08T18:42:00,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1327d49ffc3d4940a574695b745aba82, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T18:42:00,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/f07882e99b9c4aea956bd7fa97c7655d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/f07882e99b9c4aea956bd7fa97c7655d 2024-11-08T18:42:00,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/f07882e99b9c4aea956bd7fa97c7655d, entries=150, sequenceid=192, filesize=11.9 K 2024-11-08T18:42:00,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 851ms, sequenceid=192, compaction requested=false 2024-11-08T18:42:00,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:00,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-08T18:42:00,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:00,344 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-08T18:42:00,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c41ffe615afc48779d8ab3ffbf7a103c is 50, key is test_row_0/A:col10/1731091319448/Put/seqid=0 2024-11-08T18:42:00,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742294_1470 (size=12151) 2024-11-08T18:42:00,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:00,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:00,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:00,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091380700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:00,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T18:42:00,755 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c41ffe615afc48779d8ab3ffbf7a103c 2024-11-08T18:42:00,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/679fbec21d3d486790c9c087cd0e9369 is 50, key is test_row_0/B:col10/1731091319448/Put/seqid=0 2024-11-08T18:42:00,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742295_1471 (size=12151) 2024-11-08T18:42:00,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091380809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:01,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091381019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:01,165 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/679fbec21d3d486790c9c087cd0e9369 2024-11-08T18:42:01,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/7b7fc32404e145f6a84963f18a73c007 is 50, key is test_row_0/C:col10/1731091319448/Put/seqid=0 2024-11-08T18:42:01,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742296_1472 (size=12151) 2024-11-08T18:42:01,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:01,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091381329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:01,336 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:01,575 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/7b7fc32404e145f6a84963f18a73c007 2024-11-08T18:42:01,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c41ffe615afc48779d8ab3ffbf7a103c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c41ffe615afc48779d8ab3ffbf7a103c 2024-11-08T18:42:01,580 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c41ffe615afc48779d8ab3ffbf7a103c, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T18:42:01,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/679fbec21d3d486790c9c087cd0e9369 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/679fbec21d3d486790c9c087cd0e9369 2024-11-08T18:42:01,584 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/679fbec21d3d486790c9c087cd0e9369, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T18:42:01,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/7b7fc32404e145f6a84963f18a73c007 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/7b7fc32404e145f6a84963f18a73c007 2024-11-08T18:42:01,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/7b7fc32404e145f6a84963f18a73c007, entries=150, sequenceid=210, filesize=11.9 K 2024-11-08T18:42:01,587 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1243ms, sequenceid=210, compaction requested=true 2024-11-08T18:42:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:01,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:01,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-08T18:42:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-08T18:42:01,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-08T18:42:01,590 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0070 sec 2024-11-08T18:42:01,591 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.0100 sec 2024-11-08T18:42:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-08T18:42:01,730 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:01,730 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-08T18:42:01,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T18:42:01,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:01,732 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:01,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:01,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T18:42:01,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:01,854 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:01,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-08T18:42:01,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/733731d2e4bf4207ba15f64604bb4e27 is 50, key is test_row_0/A:col10/1731091321853/Put/seqid=0 2024-11-08T18:42:01,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742297_1473 (size=14541) 2024-11-08T18:42:01,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T18:42:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:01,884 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:01,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:01,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:01,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091381910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:01,913 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:02,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:02,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091382019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:02,025 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:02,036 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T18:42:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T18:42:02,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T18:42:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,189 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,189 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091382230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:02,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:02,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/733731d2e4bf4207ba15f64604bb4e27 2024-11-08T18:42:02,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/e32ed7567d8f4ced964581a80bf4c158 is 50, key is test_row_0/B:col10/1731091321853/Put/seqid=0 2024-11-08T18:42:02,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742298_1474 (size=12151) 2024-11-08T18:42:02,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/e32ed7567d8f4ced964581a80bf4c158 2024-11-08T18:42:02,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/a654c60687ea4e1dadf5dd46f2d489ee is 50, key is test_row_0/C:col10/1731091321853/Put/seqid=0 2024-11-08T18:42:02,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742299_1475 (size=12151) 2024-11-08T18:42:02,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T18:42:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,342 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T18:42:02,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T18:42:02,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:02,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:02,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091382539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:02,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:02,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T18:42:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:02,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/a654c60687ea4e1dadf5dd46f2d489ee 2024-11-08T18:42:02,688 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/733731d2e4bf4207ba15f64604bb4e27 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/733731d2e4bf4207ba15f64604bb4e27 2024-11-08T18:42:02,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/733731d2e4bf4207ba15f64604bb4e27, entries=200, sequenceid=232, filesize=14.2 K 2024-11-08T18:42:02,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/e32ed7567d8f4ced964581a80bf4c158 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e32ed7567d8f4ced964581a80bf4c158 2024-11-08T18:42:02,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e32ed7567d8f4ced964581a80bf4c158, entries=150, sequenceid=232, filesize=11.9 K 2024-11-08T18:42:02,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/a654c60687ea4e1dadf5dd46f2d489ee as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a654c60687ea4e1dadf5dd46f2d489ee 2024-11-08T18:42:02,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a654c60687ea4e1dadf5dd46f2d489ee, entries=150, sequenceid=232, filesize=11.9 K 2024-11-08T18:42:02,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 843ms, sequenceid=232, compaction requested=true 2024-11-08T18:42:02,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:02,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:02,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:02,698 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:42:02,698 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:42:02,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:02,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:02,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:02,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:02,698 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:42:02,698 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:42:02,698 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/B is initiating minor compaction (all files) 2024-11-08T18:42:02,698 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/A is initiating minor compaction (all files) 2024-11-08T18:42:02,698 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/A in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,698 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/B in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,699 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f3d5243d11fe4a83836e5075d46d2b2f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f084ef0eb815452b9ecc9db270e17f0f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c41ffe615afc48779d8ab3ffbf7a103c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/733731d2e4bf4207ba15f64604bb4e27] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=52.5 K 2024-11-08T18:42:02,699 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a554df30737d424e9417777477534c26, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1327d49ffc3d4940a574695b745aba82, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/679fbec21d3d486790c9c087cd0e9369, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e32ed7567d8f4ced964581a80bf4c158] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=47.9 K 2024-11-08T18:42:02,699 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3d5243d11fe4a83836e5075d46d2b2f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091316934 2024-11-08T18:42:02,699 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting a554df30737d424e9417777477534c26, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091316934 2024-11-08T18:42:02,699 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 1327d49ffc3d4940a574695b745aba82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731091318132 2024-11-08T18:42:02,699 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f084ef0eb815452b9ecc9db270e17f0f, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731091318132 2024-11-08T18:42:02,699 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 679fbec21d3d486790c9c087cd0e9369, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731091319429 2024-11-08T18:42:02,699 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c41ffe615afc48779d8ab3ffbf7a103c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731091319429 2024-11-08T18:42:02,699 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting e32ed7567d8f4ced964581a80bf4c158, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091320672 2024-11-08T18:42:02,699 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 733731d2e4bf4207ba15f64604bb4e27, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091320672 2024-11-08T18:42:02,707 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#A#compaction#402 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:02,707 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#B#compaction#403 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:02,707 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/623616cc3e874cdaa317e95e59b6c0be is 50, key is test_row_0/A:col10/1731091321853/Put/seqid=0 2024-11-08T18:42:02,707 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/8ea9253980b34277afe9e865304e7e93 is 50, key is test_row_0/B:col10/1731091321853/Put/seqid=0 2024-11-08T18:42:02,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742300_1476 (size=12697) 2024-11-08T18:42:02,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742301_1477 (size=12697) 2024-11-08T18:42:02,720 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/623616cc3e874cdaa317e95e59b6c0be as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/623616cc3e874cdaa317e95e59b6c0be 2024-11-08T18:42:02,725 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/A of 59c8200f26f4f9ab5bce4184c7b0cb59 into 623616cc3e874cdaa317e95e59b6c0be(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:02,725 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:02,725 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/A, priority=12, startTime=1731091322697; duration=0sec 2024-11-08T18:42:02,725 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:02,725 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:A 2024-11-08T18:42:02,725 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-08T18:42:02,726 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-08T18:42:02,726 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/C is initiating minor compaction (all files) 2024-11-08T18:42:02,726 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/C in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,726 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5aaf5a17fe934733ba4e42ed105bb5a5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/f07882e99b9c4aea956bd7fa97c7655d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/7b7fc32404e145f6a84963f18a73c007, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a654c60687ea4e1dadf5dd46f2d489ee] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=47.9 K 2024-11-08T18:42:02,726 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aaf5a17fe934733ba4e42ed105bb5a5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731091316934 2024-11-08T18:42:02,726 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f07882e99b9c4aea956bd7fa97c7655d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731091318132 2024-11-08T18:42:02,726 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b7fc32404e145f6a84963f18a73c007, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731091319429 2024-11-08T18:42:02,727 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a654c60687ea4e1dadf5dd46f2d489ee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091320672 2024-11-08T18:42:02,733 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#C#compaction#404 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:02,733 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/170d6572422b457e8fb0fe2357229c36 is 50, key is test_row_0/C:col10/1731091321853/Put/seqid=0 2024-11-08T18:42:02,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742302_1478 (size=12697) 2024-11-08T18:42:02,743 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/170d6572422b457e8fb0fe2357229c36 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/170d6572422b457e8fb0fe2357229c36 2024-11-08T18:42:02,746 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/C of 59c8200f26f4f9ab5bce4184c7b0cb59 into 170d6572422b457e8fb0fe2357229c36(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:02,746 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:02,746 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/C, priority=12, startTime=1731091322698; duration=0sec 2024-11-08T18:42:02,746 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:02,746 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:C 2024-11-08T18:42:02,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-08T18:42:02,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:02,800 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:42:02,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/59d7b627dea249638cd3c800991746b0 is 50, key is test_row_0/A:col10/1731091321901/Put/seqid=0 2024-11-08T18:42:02,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742303_1479 (size=12151) 2024-11-08T18:42:02,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T18:42:03,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:03,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:03,119 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/8ea9253980b34277afe9e865304e7e93 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8ea9253980b34277afe9e865304e7e93 2024-11-08T18:42:03,123 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/B of 59c8200f26f4f9ab5bce4184c7b0cb59 into 8ea9253980b34277afe9e865304e7e93(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:03,123 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:03,123 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/B, priority=12, startTime=1731091322698; duration=0sec 2024-11-08T18:42:03,123 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:03,123 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:B 2024-11-08T18:42:03,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:03,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091383148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,156 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:03,207 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/59d7b627dea249638cd3c800991746b0 2024-11-08T18:42:03,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/13669e1e86cb4bd4904f329a17bbafeb is 50, key is test_row_0/B:col10/1731091321901/Put/seqid=0 2024-11-08T18:42:03,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742304_1480 (size=12151) 2024-11-08T18:42:03,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:03,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091383259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:03,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:03,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091383479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:03,615 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/13669e1e86cb4bd4904f329a17bbafeb 2024-11-08T18:42:03,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/3d844e0c26a04208be50b0ddd243a9e3 is 50, key is test_row_0/C:col10/1731091321901/Put/seqid=0 2024-11-08T18:42:03,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742305_1481 (size=12151) 2024-11-08T18:42:03,755 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33570 deadline: 1731091383749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:03,755 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:42:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,755 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:03,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:03,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33556 deadline: 1731091383769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:03,773 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:42:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:03,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:03,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33534 deadline: 1731091383779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:03,786 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8257 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:42:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:03,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33594 deadline: 1731091383790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:03,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:03,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091383790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:03,794 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_1' in 59c8200f26f4f9ab5bce4184c7b0cb59 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8265 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:42:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:03,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:03,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T18:42:04,024 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/3d844e0c26a04208be50b0ddd243a9e3 2024-11-08T18:42:04,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/59d7b627dea249638cd3c800991746b0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/59d7b627dea249638cd3c800991746b0 2024-11-08T18:42:04,030 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/59d7b627dea249638cd3c800991746b0, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T18:42:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/13669e1e86cb4bd4904f329a17bbafeb as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/13669e1e86cb4bd4904f329a17bbafeb 2024-11-08T18:42:04,033 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/13669e1e86cb4bd4904f329a17bbafeb, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T18:42:04,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/3d844e0c26a04208be50b0ddd243a9e3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/3d844e0c26a04208be50b0ddd243a9e3 2024-11-08T18:42:04,037 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/3d844e0c26a04208be50b0ddd243a9e3, entries=150, sequenceid=248, filesize=11.9 K 2024-11-08T18:42:04,038 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1239ms, sequenceid=248, compaction requested=false 2024-11-08T18:42:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:04,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-08T18:42:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-08T18:42:04,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-08T18:42:04,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3070 sec 2024-11-08T18:42:04,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3110 sec 2024-11-08T18:42:04,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:04,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:04,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:42:04,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/dfdcaf0a7cba425e8e707f0d393ab8c6 is 50, key is test_row_0/A:col10/1731091323138/Put/seqid=0 2024-11-08T18:42:04,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742306_1482 (size=14741) 2024-11-08T18:42:04,314 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/dfdcaf0a7cba425e8e707f0d393ab8c6 2024-11-08T18:42:04,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/6bf9effaaf85412da50216021072631a is 50, key is test_row_0/B:col10/1731091323138/Put/seqid=0 2024-11-08T18:42:04,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742307_1483 (size=12301) 2024-11-08T18:42:04,358 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:04,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091384352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:04,358 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:04,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:04,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091384470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:04,477 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:04,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:04,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091384679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:04,687 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:04,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/6bf9effaaf85412da50216021072631a 2024-11-08T18:42:04,729 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/283c8ae63b1d481eb8a62a8a92a63956 is 50, key is test_row_0/C:col10/1731091323138/Put/seqid=0 2024-11-08T18:42:04,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742308_1484 (size=12301) 2024-11-08T18:42:04,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/283c8ae63b1d481eb8a62a8a92a63956 2024-11-08T18:42:04,738 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/dfdcaf0a7cba425e8e707f0d393ab8c6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/dfdcaf0a7cba425e8e707f0d393ab8c6 2024-11-08T18:42:04,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/dfdcaf0a7cba425e8e707f0d393ab8c6, entries=200, sequenceid=272, filesize=14.4 K 2024-11-08T18:42:04,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/6bf9effaaf85412da50216021072631a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/6bf9effaaf85412da50216021072631a 2024-11-08T18:42:04,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/6bf9effaaf85412da50216021072631a, entries=150, sequenceid=272, filesize=12.0 K 2024-11-08T18:42:04,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/283c8ae63b1d481eb8a62a8a92a63956 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/283c8ae63b1d481eb8a62a8a92a63956 2024-11-08T18:42:04,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/283c8ae63b1d481eb8a62a8a92a63956, entries=150, sequenceid=272, filesize=12.0 K 2024-11-08T18:42:04,747 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 442ms, sequenceid=272, compaction requested=true 2024-11-08T18:42:04,747 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:04,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:04,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:04,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:04,748 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:04,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:04,748 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:04,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:04,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:04,748 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:04,748 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:04,748 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/A is initiating minor compaction (all files) 2024-11-08T18:42:04,748 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/B is initiating minor compaction (all files) 2024-11-08T18:42:04,748 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/A in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:04,748 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/B in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:04,749 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/623616cc3e874cdaa317e95e59b6c0be, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/59d7b627dea249638cd3c800991746b0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/dfdcaf0a7cba425e8e707f0d393ab8c6] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=38.7 K 2024-11-08T18:42:04,749 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8ea9253980b34277afe9e865304e7e93, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/13669e1e86cb4bd4904f329a17bbafeb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/6bf9effaaf85412da50216021072631a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=36.3 K 2024-11-08T18:42:04,749 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ea9253980b34277afe9e865304e7e93, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091320672 2024-11-08T18:42:04,749 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 623616cc3e874cdaa317e95e59b6c0be, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091320672 2024-11-08T18:42:04,749 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 13669e1e86cb4bd4904f329a17bbafeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731091321880 2024-11-08T18:42:04,749 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59d7b627dea249638cd3c800991746b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731091321880 2024-11-08T18:42:04,749 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfdcaf0a7cba425e8e707f0d393ab8c6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731091323138 2024-11-08T18:42:04,749 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bf9effaaf85412da50216021072631a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731091323138 2024-11-08T18:42:04,755 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#A#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:04,755 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/3a101e597d874ca682ed3bae6767f9c0 is 50, key is test_row_0/A:col10/1731091323138/Put/seqid=0 2024-11-08T18:42:04,758 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#B#compaction#412 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:04,758 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/d29cbf1c04014f169e74cba6d44fb28a is 50, key is test_row_0/B:col10/1731091323138/Put/seqid=0 2024-11-08T18:42:04,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742309_1485 (size=12949) 2024-11-08T18:42:04,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742310_1486 (size=12949) 2024-11-08T18:42:05,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:05,005 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:05,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:42:05,009 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/9814aaec02ed47549a99d083d2a1184e is 50, key is test_row_0/A:col10/1731091324351/Put/seqid=0 2024-11-08T18:42:05,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742311_1487 (size=14741) 2024-11-08T18:42:05,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:05,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091385151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:05,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:05,176 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/3a101e597d874ca682ed3bae6767f9c0 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/3a101e597d874ca682ed3bae6767f9c0 2024-11-08T18:42:05,176 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/d29cbf1c04014f169e74cba6d44fb28a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/d29cbf1c04014f169e74cba6d44fb28a 2024-11-08T18:42:05,179 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/B of 59c8200f26f4f9ab5bce4184c7b0cb59 into d29cbf1c04014f169e74cba6d44fb28a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:05,179 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/A of 59c8200f26f4f9ab5bce4184c7b0cb59 into 3a101e597d874ca682ed3bae6767f9c0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:05,179 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:05,179 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:05,179 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/B, priority=13, startTime=1731091324748; duration=0sec 2024-11-08T18:42:05,179 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/A, priority=13, startTime=1731091324748; duration=0sec 2024-11-08T18:42:05,179 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:05,179 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:B 2024-11-08T18:42:05,179 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:05,179 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:A 2024-11-08T18:42:05,179 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:05,180 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:05,180 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/C is initiating minor compaction (all files) 2024-11-08T18:42:05,180 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/C in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:05,180 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/170d6572422b457e8fb0fe2357229c36, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/3d844e0c26a04208be50b0ddd243a9e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/283c8ae63b1d481eb8a62a8a92a63956] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=36.3 K 2024-11-08T18:42:05,180 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 170d6572422b457e8fb0fe2357229c36, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731091320672 2024-11-08T18:42:05,180 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d844e0c26a04208be50b0ddd243a9e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731091321880 2024-11-08T18:42:05,181 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 283c8ae63b1d481eb8a62a8a92a63956, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731091323138 2024-11-08T18:42:05,186 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#C#compaction#414 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:05,186 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/c4c6a34d621b462e83944857dad94735 is 50, key is test_row_0/C:col10/1731091323138/Put/seqid=0 2024-11-08T18:42:05,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742312_1488 (size=12949) 2024-11-08T18:42:05,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091385270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:05,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/9814aaec02ed47549a99d083d2a1184e 2024-11-08T18:42:05,422 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/16f63533698f43b4a04e7399986d03ce is 50, key is test_row_0/B:col10/1731091324351/Put/seqid=0 2024-11-08T18:42:05,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742313_1489 (size=12301) 2024-11-08T18:42:05,484 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:05,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091385479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:05,484 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:05,593 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/c4c6a34d621b462e83944857dad94735 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/c4c6a34d621b462e83944857dad94735 2024-11-08T18:42:05,597 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/C of 59c8200f26f4f9ab5bce4184c7b0cb59 into c4c6a34d621b462e83944857dad94735(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:05,597 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:05,597 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/C, priority=13, startTime=1731091324748; duration=0sec 2024-11-08T18:42:05,597 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:05,597 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:C 2024-11-08T18:42:05,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:05,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091385789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:05,792 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:05,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/16f63533698f43b4a04e7399986d03ce 2024-11-08T18:42:05,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e8da5a2a956b44939d0d59f78d399f63 is 50, key is test_row_0/C:col10/1731091324351/Put/seqid=0 2024-11-08T18:42:05,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742314_1490 (size=12301) 2024-11-08T18:42:05,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-08T18:42:05,900 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:05,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-08T18:42:05,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T18:42:05,901 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:05,902 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:05,902 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:06,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T18:42:06,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:06,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:06,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T18:42:06,234 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e8da5a2a956b44939d0d59f78d399f63 2024-11-08T18:42:06,237 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/9814aaec02ed47549a99d083d2a1184e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9814aaec02ed47549a99d083d2a1184e 2024-11-08T18:42:06,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9814aaec02ed47549a99d083d2a1184e, entries=200, sequenceid=286, filesize=14.4 K 2024-11-08T18:42:06,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/16f63533698f43b4a04e7399986d03ce as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/16f63533698f43b4a04e7399986d03ce 2024-11-08T18:42:06,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/16f63533698f43b4a04e7399986d03ce, entries=150, sequenceid=286, filesize=12.0 K 2024-11-08T18:42:06,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e8da5a2a956b44939d0d59f78d399f63 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e8da5a2a956b44939d0d59f78d399f63 2024-11-08T18:42:06,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e8da5a2a956b44939d0d59f78d399f63, entries=150, sequenceid=286, filesize=12.0 K 2024-11-08T18:42:06,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1242ms, sequenceid=286, compaction requested=false 2024-11-08T18:42:06,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:06,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:06,305 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:06,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-08T18:42:06,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c8b56e5ebded4f659a245da93f59133f is 50, key is test_row_0/A:col10/1731091325120/Put/seqid=0 2024-11-08T18:42:06,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742315_1491 (size=14741) 2024-11-08T18:42:06,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:06,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:06,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091386358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:06,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:06,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091386469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:06,476 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:06,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:06,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:06,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T18:42:06,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:06,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:06,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091386679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:06,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:06,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c8b56e5ebded4f659a245da93f59133f 2024-11-08T18:42:06,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c777418f5cf047cc840e509385bfa324 is 50, key is test_row_0/B:col10/1731091325120/Put/seqid=0 2024-11-08T18:42:06,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742316_1492 (size=12301) 2024-11-08T18:42:06,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:06,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:06,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,901 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-11-08T18:42:06,901 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:06,901 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,901 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:06,902 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-08T18:42:06,902 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:06,903 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,903 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:06,906 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-08T18:42:06,906 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:06,906 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,906 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,907 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:06,910 INFO [Thread-1947 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1947. 2024-11-08T18:42:06,910 DEBUG [Thread-1947 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:06,910 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,910 DEBUG [Thread-1947 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,910 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:06,912 INFO [Thread-1939 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1939. 2024-11-08T18:42:06,912 DEBUG [Thread-1939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:06,912 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,912 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:06,912 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:06,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:06,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:06,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:06,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091386989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:06,991 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:07,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T18:42:07,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c777418f5cf047cc840e509385bfa324 2024-11-08T18:42:07,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:07,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e1c41c9f3e944683a53508d1c1972d8c is 50, key is test_row_0/C:col10/1731091325120/Put/seqid=0 2024-11-08T18:42:07,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742317_1493 (size=12301) 2024-11-08T18:42:07,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:07,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:07,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:07,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:07,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33580 deadline: 1731091387500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 , the old value is region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:07,501 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., hostname=b75fe890c219,44447,1731091183487, seqNum=2 because the exception is null or not the one we care about 2024-11-08T18:42:07,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e1c41c9f3e944683a53508d1c1972d8c 2024-11-08T18:42:07,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/c8b56e5ebded4f659a245da93f59133f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c8b56e5ebded4f659a245da93f59133f 2024-11-08T18:42:07,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c8b56e5ebded4f659a245da93f59133f, entries=200, sequenceid=312, filesize=14.4 K 2024-11-08T18:42:07,554 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/c777418f5cf047cc840e509385bfa324 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c777418f5cf047cc840e509385bfa324 2024-11-08T18:42:07,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c777418f5cf047cc840e509385bfa324, entries=150, sequenceid=312, filesize=12.0 K 2024-11-08T18:42:07,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e1c41c9f3e944683a53508d1c1972d8c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e1c41c9f3e944683a53508d1c1972d8c 2024-11-08T18:42:07,563 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e1c41c9f3e944683a53508d1c1972d8c, entries=150, sequenceid=312, filesize=12.0 K 2024-11-08T18:42:07,564 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1259ms, sequenceid=312, compaction requested=true 2024-11-08T18:42:07,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:07,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:07,564 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:07,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:07,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:07,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:07,564 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:07,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 59c8200f26f4f9ab5bce4184c7b0cb59:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:07,564 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:07,565 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:07,565 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:07,565 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/A is initiating minor compaction (all files) 2024-11-08T18:42:07,565 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/B is initiating minor compaction (all files) 2024-11-08T18:42:07,565 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/A in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,565 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/B in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,566 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/3a101e597d874ca682ed3bae6767f9c0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9814aaec02ed47549a99d083d2a1184e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c8b56e5ebded4f659a245da93f59133f] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=41.4 K 2024-11-08T18:42:07,566 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/d29cbf1c04014f169e74cba6d44fb28a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/16f63533698f43b4a04e7399986d03ce, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c777418f5cf047cc840e509385bfa324] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=36.7 K 2024-11-08T18:42:07,566 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d29cbf1c04014f169e74cba6d44fb28a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731091323138 2024-11-08T18:42:07,566 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a101e597d874ca682ed3bae6767f9c0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731091323138 2024-11-08T18:42:07,566 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9814aaec02ed47549a99d083d2a1184e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731091324331 2024-11-08T18:42:07,566 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 16f63533698f43b4a04e7399986d03ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731091324347 2024-11-08T18:42:07,567 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8b56e5ebded4f659a245da93f59133f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731091325120 2024-11-08T18:42:07,567 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting c777418f5cf047cc840e509385bfa324, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731091325120 2024-11-08T18:42:07,573 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#B#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:07,574 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#A#compaction#421 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:07,574 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/cc25178018fb41f89543311d53a50535 is 50, key is test_row_0/B:col10/1731091325120/Put/seqid=0 2024-11-08T18:42:07,574 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/ec5e48fd12eb4867831821680bb71d5c is 50, key is test_row_0/A:col10/1731091325120/Put/seqid=0 2024-11-08T18:42:07,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742318_1494 (size=13051) 2024-11-08T18:42:07,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742319_1495 (size=13051) 2024-11-08T18:42:07,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-08T18:42:07,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,595 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:42:07,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/0aff1cbbacbd4abd98d61f009aeaf993 is 50, key is test_row_0/A:col10/1731091326334/Put/seqid=0 2024-11-08T18:42:07,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742320_1496 (size=12301) 2024-11-08T18:42:07,987 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/cc25178018fb41f89543311d53a50535 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/cc25178018fb41f89543311d53a50535 2024-11-08T18:42:07,987 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/ec5e48fd12eb4867831821680bb71d5c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/ec5e48fd12eb4867831821680bb71d5c 2024-11-08T18:42:07,994 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/A of 59c8200f26f4f9ab5bce4184c7b0cb59 into ec5e48fd12eb4867831821680bb71d5c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:07,994 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/B of 59c8200f26f4f9ab5bce4184c7b0cb59 into cc25178018fb41f89543311d53a50535(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:07,994 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:07,994 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:07,995 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/B, priority=13, startTime=1731091327564; duration=0sec 2024-11-08T18:42:07,995 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/A, priority=13, startTime=1731091327564; duration=0sec 2024-11-08T18:42:07,995 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:07,995 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:07,995 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:A 2024-11-08T18:42:07,995 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:B 2024-11-08T18:42:07,995 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:07,996 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:07,996 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): 59c8200f26f4f9ab5bce4184c7b0cb59/C is initiating minor compaction (all files) 2024-11-08T18:42:07,996 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 59c8200f26f4f9ab5bce4184c7b0cb59/C in TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:07,997 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/c4c6a34d621b462e83944857dad94735, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e8da5a2a956b44939d0d59f78d399f63, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e1c41c9f3e944683a53508d1c1972d8c] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp, totalSize=36.7 K 2024-11-08T18:42:07,997 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4c6a34d621b462e83944857dad94735, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731091323138 2024-11-08T18:42:07,997 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8da5a2a956b44939d0d59f78d399f63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731091324347 2024-11-08T18:42:07,997 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1c41c9f3e944683a53508d1c1972d8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731091325120 2024-11-08T18:42:08,002 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/0aff1cbbacbd4abd98d61f009aeaf993 2024-11-08T18:42:08,004 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 59c8200f26f4f9ab5bce4184c7b0cb59#C#compaction#423 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:08,004 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/db35719ba3fb4a0c969abb13a05914eb is 50, key is test_row_0/C:col10/1731091325120/Put/seqid=0 2024-11-08T18:42:08,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742321_1497 (size=13051) 2024-11-08T18:42:08,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1a2c12ddfff3489db9f7ece7fd486d57 is 50, key is test_row_0/B:col10/1731091326334/Put/seqid=0 2024-11-08T18:42:08,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742322_1498 (size=12301) 2024-11-08T18:42:08,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T18:42:08,413 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1a2c12ddfff3489db9f7ece7fd486d57 2024-11-08T18:42:08,420 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/db35719ba3fb4a0c969abb13a05914eb as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/db35719ba3fb4a0c969abb13a05914eb 2024-11-08T18:42:08,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/b92d1660de90444dbe329af9e54968c6 is 50, key is test_row_0/C:col10/1731091326334/Put/seqid=0 2024-11-08T18:42:08,426 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 59c8200f26f4f9ab5bce4184c7b0cb59/C of 59c8200f26f4f9ab5bce4184c7b0cb59 into db35719ba3fb4a0c969abb13a05914eb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:08,426 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:08,426 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., storeName=59c8200f26f4f9ab5bce4184c7b0cb59/C, priority=13, startTime=1731091327564; duration=0sec 2024-11-08T18:42:08,426 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:08,426 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 59c8200f26f4f9ab5bce4184c7b0cb59:C 2024-11-08T18:42:08,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742323_1499 (size=12301) 2024-11-08T18:42:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:08,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. as already flushing 2024-11-08T18:42:08,512 INFO [Thread-1936 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1936. 2024-11-08T18:42:08,512 DEBUG [Thread-1936 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:08,512 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:08,512 DEBUG [Thread-1936 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:08,512 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:08,830 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/b92d1660de90444dbe329af9e54968c6 2024-11-08T18:42:08,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/0aff1cbbacbd4abd98d61f009aeaf993 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/0aff1cbbacbd4abd98d61f009aeaf993 2024-11-08T18:42:08,846 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/0aff1cbbacbd4abd98d61f009aeaf993, entries=150, sequenceid=325, filesize=12.0 K 2024-11-08T18:42:08,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/1a2c12ddfff3489db9f7ece7fd486d57 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1a2c12ddfff3489db9f7ece7fd486d57 2024-11-08T18:42:08,852 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1a2c12ddfff3489db9f7ece7fd486d57, entries=150, sequenceid=325, filesize=12.0 K 2024-11-08T18:42:08,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/b92d1660de90444dbe329af9e54968c6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/b92d1660de90444dbe329af9e54968c6 2024-11-08T18:42:08,857 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/b92d1660de90444dbe329af9e54968c6, entries=150, sequenceid=325, filesize=12.0 K 2024-11-08T18:42:08,858 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=6.71 KB/6870 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1263ms, sequenceid=325, compaction requested=false 2024-11-08T18:42:08,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 59c8200f26f4f9ab5bce4184c7b0cb59: 2024-11-08T18:42:08,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:08,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-08T18:42:08,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-08T18:42:08,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-08T18:42:08,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9580 sec 2024-11-08T18:42:08,863 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.9620 sec 2024-11-08T18:42:10,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-08T18:42:10,061 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:11,639 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T18:42:13,824 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-11-08T18:42:13,824 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:13,824 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,825 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:13,843 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-11-08T18:42:13,843 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-11-08T18:42:13,844 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:13,844 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:13,844 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,844 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,844 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,844 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:13,844 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:13,852 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-08T18:42:13,852 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:13,852 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,852 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 27 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 26 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 156 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1501 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4496 rows 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1494 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4481 rows 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1488 2024-11-08T18:42:13,852 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4463 rows 2024-11-08T18:42:13,853 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1489 2024-11-08T18:42:13,853 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4466 rows 2024-11-08T18:42:13,853 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1496 2024-11-08T18:42:13,853 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4487 rows 2024-11-08T18:42:13,853 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T18:42:13,853 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:42:13,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,853 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:13,853 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:13,854 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T18:42:13,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T18:42:13,858 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091333857"}]},"ts":"1731091333857"} 2024-11-08T18:42:13,859 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T18:42:13,859 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T18:42:13,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:42:13,862 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, UNASSIGN}] 2024-11-08T18:42:13,863 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, UNASSIGN 2024-11-08T18:42:13,864 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=59c8200f26f4f9ab5bce4184c7b0cb59, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:42:13,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, UNASSIGN because future has completed 2024-11-08T18:42:13,866 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:42:13,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:42:13,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T18:42:14,021 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:14,021 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:42:14,021 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 59c8200f26f4f9ab5bce4184c7b0cb59, disabling compactions & flushes 2024-11-08T18:42:14,021 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:14,021 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:14,021 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. after waiting 0 ms 2024-11-08T18:42:14,022 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:14,022 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 59c8200f26f4f9ab5bce4184c7b0cb59 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T18:42:14,029 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/881a87b609ea41acb9ba71cc7c5bfccf is 50, key is test_row_1/A:col10/1731091333850/Put/seqid=0 2024-11-08T18:42:14,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742324_1500 (size=7415) 2024-11-08T18:42:14,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T18:42:14,434 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/881a87b609ea41acb9ba71cc7c5bfccf 2024-11-08T18:42:14,448 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/912683af5a424fdb8a2cf61d0539b55e is 50, key is test_row_1/B:col10/1731091333850/Put/seqid=0 2024-11-08T18:42:14,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742325_1501 (size=7415) 2024-11-08T18:42:14,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T18:42:14,853 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/912683af5a424fdb8a2cf61d0539b55e 2024-11-08T18:42:14,866 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e405580bb6b3456f923b59ed5e3d8807 is 50, key is test_row_1/C:col10/1731091333850/Put/seqid=0 2024-11-08T18:42:14,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742326_1502 (size=7415) 2024-11-08T18:42:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T18:42:15,270 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e405580bb6b3456f923b59ed5e3d8807 2024-11-08T18:42:15,277 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/A/881a87b609ea41acb9ba71cc7c5bfccf as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/881a87b609ea41acb9ba71cc7c5bfccf 2024-11-08T18:42:15,280 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/881a87b609ea41acb9ba71cc7c5bfccf, entries=50, sequenceid=336, filesize=7.2 K 2024-11-08T18:42:15,281 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/B/912683af5a424fdb8a2cf61d0539b55e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/912683af5a424fdb8a2cf61d0539b55e 2024-11-08T18:42:15,284 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/912683af5a424fdb8a2cf61d0539b55e, entries=50, sequenceid=336, filesize=7.2 K 2024-11-08T18:42:15,285 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/.tmp/C/e405580bb6b3456f923b59ed5e3d8807 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e405580bb6b3456f923b59ed5e3d8807 2024-11-08T18:42:15,288 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e405580bb6b3456f923b59ed5e3d8807, entries=50, sequenceid=336, filesize=7.2 K 2024-11-08T18:42:15,288 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1266ms, sequenceid=336, compaction requested=true 2024-11-08T18:42:15,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/de5d1ee5273247f886cd20d118c83796, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/5fb5002cceea44ea92883c4cb3ffaf8d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f8892ace61fd4ab1ad04819860aee9fd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/2e81027587824d528a81db0265167477, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/b4c131dffc964011a119c9a4cf68944c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f93315ed71534935a7f6bbe37c2cbaba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/654c628d55814bc4a60d20b522582ecc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/cb882d763bc649dc99451f814adf2e0b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/09b6c893b73a461cb7fe5da4a55a8623, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/517a2fe9b4004d1c994178d06862afb6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c22d1aec4e42403a9a15759d1115d26e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9d0c01c6161f41fc92ed8160dbcb03c5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f3d5243d11fe4a83836e5075d46d2b2f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f084ef0eb815452b9ecc9db270e17f0f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c41ffe615afc48779d8ab3ffbf7a103c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/733731d2e4bf4207ba15f64604bb4e27, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/623616cc3e874cdaa317e95e59b6c0be, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/59d7b627dea249638cd3c800991746b0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/dfdcaf0a7cba425e8e707f0d393ab8c6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/3a101e597d874ca682ed3bae6767f9c0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9814aaec02ed47549a99d083d2a1184e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c8b56e5ebded4f659a245da93f59133f] to archive 2024-11-08T18:42:15,290 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:42:15,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/de5d1ee5273247f886cd20d118c83796 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/de5d1ee5273247f886cd20d118c83796 2024-11-08T18:42:15,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/5fb5002cceea44ea92883c4cb3ffaf8d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/5fb5002cceea44ea92883c4cb3ffaf8d 2024-11-08T18:42:15,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f8892ace61fd4ab1ad04819860aee9fd to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f8892ace61fd4ab1ad04819860aee9fd 2024-11-08T18:42:15,294 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/2e81027587824d528a81db0265167477 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/2e81027587824d528a81db0265167477 2024-11-08T18:42:15,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/b4c131dffc964011a119c9a4cf68944c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/b4c131dffc964011a119c9a4cf68944c 2024-11-08T18:42:15,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f93315ed71534935a7f6bbe37c2cbaba to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f93315ed71534935a7f6bbe37c2cbaba 2024-11-08T18:42:15,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/654c628d55814bc4a60d20b522582ecc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/654c628d55814bc4a60d20b522582ecc 2024-11-08T18:42:15,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/cb882d763bc649dc99451f814adf2e0b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/cb882d763bc649dc99451f814adf2e0b 2024-11-08T18:42:15,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/09b6c893b73a461cb7fe5da4a55a8623 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/09b6c893b73a461cb7fe5da4a55a8623 2024-11-08T18:42:15,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/517a2fe9b4004d1c994178d06862afb6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/517a2fe9b4004d1c994178d06862afb6 2024-11-08T18:42:15,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c22d1aec4e42403a9a15759d1115d26e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c22d1aec4e42403a9a15759d1115d26e 2024-11-08T18:42:15,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9d0c01c6161f41fc92ed8160dbcb03c5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9d0c01c6161f41fc92ed8160dbcb03c5 2024-11-08T18:42:15,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f3d5243d11fe4a83836e5075d46d2b2f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f3d5243d11fe4a83836e5075d46d2b2f 2024-11-08T18:42:15,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f084ef0eb815452b9ecc9db270e17f0f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/f084ef0eb815452b9ecc9db270e17f0f 2024-11-08T18:42:15,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c41ffe615afc48779d8ab3ffbf7a103c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c41ffe615afc48779d8ab3ffbf7a103c 2024-11-08T18:42:15,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/733731d2e4bf4207ba15f64604bb4e27 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/733731d2e4bf4207ba15f64604bb4e27 2024-11-08T18:42:15,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/623616cc3e874cdaa317e95e59b6c0be to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/623616cc3e874cdaa317e95e59b6c0be 2024-11-08T18:42:15,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/59d7b627dea249638cd3c800991746b0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/59d7b627dea249638cd3c800991746b0 2024-11-08T18:42:15,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/dfdcaf0a7cba425e8e707f0d393ab8c6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/dfdcaf0a7cba425e8e707f0d393ab8c6 2024-11-08T18:42:15,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/3a101e597d874ca682ed3bae6767f9c0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/3a101e597d874ca682ed3bae6767f9c0 2024-11-08T18:42:15,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9814aaec02ed47549a99d083d2a1184e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/9814aaec02ed47549a99d083d2a1184e 2024-11-08T18:42:15,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c8b56e5ebded4f659a245da93f59133f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/c8b56e5ebded4f659a245da93f59133f 2024-11-08T18:42:15,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/ba0da29381ab4bc09485e7977c66e642, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c4429e70727c409ba09c09f7ea3bd3ba, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/3d5ce44cc962452cbaaa971b19879066, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/5305fa8a131d49839777d86ee23289f0, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1e2529f905514ec5926234db0c43bfbb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/2649588d531047fdbf2b545c574dbd5d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c25eee6303044a95b1f6bd3896730d67, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/9b1a098f615f41eb89c7fee7860ac26e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a1f23c9c70eb4802ae26bbdf1b36a517, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8603e7a9af5f46cbab5644d4cd84e94a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e06c3bfe5f6f4723a47f5871a7d7d457, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a554df30737d424e9417777477534c26, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1327d49ffc3d4940a574695b745aba82, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/679fbec21d3d486790c9c087cd0e9369, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8ea9253980b34277afe9e865304e7e93, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e32ed7567d8f4ced964581a80bf4c158, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/13669e1e86cb4bd4904f329a17bbafeb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/d29cbf1c04014f169e74cba6d44fb28a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/6bf9effaaf85412da50216021072631a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/16f63533698f43b4a04e7399986d03ce, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c777418f5cf047cc840e509385bfa324] to archive 2024-11-08T18:42:15,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:42:15,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/ba0da29381ab4bc09485e7977c66e642 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/ba0da29381ab4bc09485e7977c66e642 2024-11-08T18:42:15,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c4429e70727c409ba09c09f7ea3bd3ba to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c4429e70727c409ba09c09f7ea3bd3ba 2024-11-08T18:42:15,310 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/3d5ce44cc962452cbaaa971b19879066 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/3d5ce44cc962452cbaaa971b19879066 2024-11-08T18:42:15,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/5305fa8a131d49839777d86ee23289f0 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/5305fa8a131d49839777d86ee23289f0 2024-11-08T18:42:15,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1e2529f905514ec5926234db0c43bfbb to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1e2529f905514ec5926234db0c43bfbb 2024-11-08T18:42:15,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/2649588d531047fdbf2b545c574dbd5d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/2649588d531047fdbf2b545c574dbd5d 2024-11-08T18:42:15,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c25eee6303044a95b1f6bd3896730d67 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c25eee6303044a95b1f6bd3896730d67 2024-11-08T18:42:15,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/9b1a098f615f41eb89c7fee7860ac26e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/9b1a098f615f41eb89c7fee7860ac26e 2024-11-08T18:42:15,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a1f23c9c70eb4802ae26bbdf1b36a517 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a1f23c9c70eb4802ae26bbdf1b36a517 2024-11-08T18:42:15,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8603e7a9af5f46cbab5644d4cd84e94a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8603e7a9af5f46cbab5644d4cd84e94a 2024-11-08T18:42:15,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e06c3bfe5f6f4723a47f5871a7d7d457 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e06c3bfe5f6f4723a47f5871a7d7d457 2024-11-08T18:42:15,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a554df30737d424e9417777477534c26 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/a554df30737d424e9417777477534c26 2024-11-08T18:42:15,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/143e2ddaf03d4bfe8b3fa603c8a7c2fd 2024-11-08T18:42:15,317 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1327d49ffc3d4940a574695b745aba82 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1327d49ffc3d4940a574695b745aba82 2024-11-08T18:42:15,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/679fbec21d3d486790c9c087cd0e9369 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/679fbec21d3d486790c9c087cd0e9369 2024-11-08T18:42:15,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8ea9253980b34277afe9e865304e7e93 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/8ea9253980b34277afe9e865304e7e93 2024-11-08T18:42:15,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e32ed7567d8f4ced964581a80bf4c158 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/e32ed7567d8f4ced964581a80bf4c158 2024-11-08T18:42:15,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/13669e1e86cb4bd4904f329a17bbafeb to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/13669e1e86cb4bd4904f329a17bbafeb 2024-11-08T18:42:15,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/d29cbf1c04014f169e74cba6d44fb28a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/d29cbf1c04014f169e74cba6d44fb28a 2024-11-08T18:42:15,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/6bf9effaaf85412da50216021072631a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/6bf9effaaf85412da50216021072631a 2024-11-08T18:42:15,321 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/16f63533698f43b4a04e7399986d03ce to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/16f63533698f43b4a04e7399986d03ce 2024-11-08T18:42:15,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c777418f5cf047cc840e509385bfa324 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/c777418f5cf047cc840e509385bfa324 2024-11-08T18:42:15,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e7e5ba0fb1524d6483ec0f376f9c7ebd, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5d374bb78da44140990f2c0f476eccef, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a60c91138e1c4460a8bde6c185edadb4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/41a8f40fee664a80805ab609b4ef9476, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/ffd30b29f81a4249a98d086d9abe81fa, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/948f8a36540f4b0ba3e72382bac950ac, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/04f1e686465045678bbaa5932ef1203f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/dbed55b44f2f4d8083322528bc96d372, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/17afde9f67d74951ac3964b7aa125c9b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/8b1d481f858f4a549478a8c9954fa9bc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/9bd95c72d84a4c41be3b389dd33e8e6b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5aaf5a17fe934733ba4e42ed105bb5a5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5ead5eabc4b1459fa7a0312428c64975, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/f07882e99b9c4aea956bd7fa97c7655d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/7b7fc32404e145f6a84963f18a73c007, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/170d6572422b457e8fb0fe2357229c36, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a654c60687ea4e1dadf5dd46f2d489ee, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/3d844e0c26a04208be50b0ddd243a9e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/c4c6a34d621b462e83944857dad94735, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/283c8ae63b1d481eb8a62a8a92a63956, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e8da5a2a956b44939d0d59f78d399f63, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e1c41c9f3e944683a53508d1c1972d8c] to archive 2024-11-08T18:42:15,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:42:15,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e7e5ba0fb1524d6483ec0f376f9c7ebd to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e7e5ba0fb1524d6483ec0f376f9c7ebd 2024-11-08T18:42:15,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5d374bb78da44140990f2c0f476eccef to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5d374bb78da44140990f2c0f476eccef 2024-11-08T18:42:15,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a60c91138e1c4460a8bde6c185edadb4 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a60c91138e1c4460a8bde6c185edadb4 2024-11-08T18:42:15,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/41a8f40fee664a80805ab609b4ef9476 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/41a8f40fee664a80805ab609b4ef9476 2024-11-08T18:42:15,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/ffd30b29f81a4249a98d086d9abe81fa to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/ffd30b29f81a4249a98d086d9abe81fa 2024-11-08T18:42:15,328 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/948f8a36540f4b0ba3e72382bac950ac to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/948f8a36540f4b0ba3e72382bac950ac 2024-11-08T18:42:15,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/04f1e686465045678bbaa5932ef1203f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/04f1e686465045678bbaa5932ef1203f 2024-11-08T18:42:15,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/dbed55b44f2f4d8083322528bc96d372 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/dbed55b44f2f4d8083322528bc96d372 2024-11-08T18:42:15,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/17afde9f67d74951ac3964b7aa125c9b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/17afde9f67d74951ac3964b7aa125c9b 2024-11-08T18:42:15,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/8b1d481f858f4a549478a8c9954fa9bc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/8b1d481f858f4a549478a8c9954fa9bc 2024-11-08T18:42:15,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/9bd95c72d84a4c41be3b389dd33e8e6b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/9bd95c72d84a4c41be3b389dd33e8e6b 2024-11-08T18:42:15,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5aaf5a17fe934733ba4e42ed105bb5a5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5aaf5a17fe934733ba4e42ed105bb5a5 2024-11-08T18:42:15,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5ead5eabc4b1459fa7a0312428c64975 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/5ead5eabc4b1459fa7a0312428c64975 2024-11-08T18:42:15,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/f07882e99b9c4aea956bd7fa97c7655d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/f07882e99b9c4aea956bd7fa97c7655d 2024-11-08T18:42:15,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/7b7fc32404e145f6a84963f18a73c007 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/7b7fc32404e145f6a84963f18a73c007 2024-11-08T18:42:15,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/170d6572422b457e8fb0fe2357229c36 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/170d6572422b457e8fb0fe2357229c36 2024-11-08T18:42:15,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a654c60687ea4e1dadf5dd46f2d489ee to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/a654c60687ea4e1dadf5dd46f2d489ee 2024-11-08T18:42:15,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/3d844e0c26a04208be50b0ddd243a9e3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/3d844e0c26a04208be50b0ddd243a9e3 2024-11-08T18:42:15,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/c4c6a34d621b462e83944857dad94735 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/c4c6a34d621b462e83944857dad94735 2024-11-08T18:42:15,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/283c8ae63b1d481eb8a62a8a92a63956 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/283c8ae63b1d481eb8a62a8a92a63956 2024-11-08T18:42:15,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e8da5a2a956b44939d0d59f78d399f63 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e8da5a2a956b44939d0d59f78d399f63 2024-11-08T18:42:15,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e1c41c9f3e944683a53508d1c1972d8c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e1c41c9f3e944683a53508d1c1972d8c 2024-11-08T18:42:15,341 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/recovered.edits/339.seqid, newMaxSeqId=339, maxSeqId=1 2024-11-08T18:42:15,341 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. 2024-11-08T18:42:15,341 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 59c8200f26f4f9ab5bce4184c7b0cb59: Waiting for close lock at 1731091334021Running coprocessor pre-close hooks at 1731091334021Disabling compacts and flushes for region at 1731091334021Disabling writes for close at 1731091334022 (+1 ms)Obtaining lock to block concurrent updates at 1731091334022Preparing flush snapshotting stores in 59c8200f26f4f9ab5bce4184c7b0cb59 at 1731091334022Finished memstore snapshotting TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731091334023 (+1 ms)Flushing stores of TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59. at 1731091334024 (+1 ms)Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/A: creating writer at 1731091334024Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/A: appending metadata at 1731091334028 (+4 ms)Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/A: closing flushed file at 1731091334028Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/B: creating writer at 1731091334444 (+416 ms)Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/B: appending metadata at 1731091334447 (+3 ms)Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/B: closing flushed file at 1731091334448 (+1 ms)Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/C: creating writer at 1731091334862 (+414 ms)Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/C: appending metadata at 1731091334865 (+3 ms)Flushing 59c8200f26f4f9ab5bce4184c7b0cb59/C: closing flushed file at 1731091334865Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@323996f8: reopening flushed file at 1731091335276 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f36c070: reopening flushed file at 1731091335281 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@27be4867: reopening flushed file at 1731091335284 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 59c8200f26f4f9ab5bce4184c7b0cb59 in 1266ms, sequenceid=336, compaction requested=true at 1731091335288 (+4 ms)Writing region close event to WAL at 1731091335338 (+50 ms)Running coprocessor post-close hooks at 1731091335341 (+3 ms)Closed at 1731091335341 2024-11-08T18:42:15,343 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:15,343 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=59c8200f26f4f9ab5bce4184c7b0cb59, regionState=CLOSED 2024-11-08T18:42:15,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:42:15,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-08T18:42:15,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 59c8200f26f4f9ab5bce4184c7b0cb59, server=b75fe890c219,44447,1731091183487 in 1.4780 sec 2024-11-08T18:42:15,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-08T18:42:15,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=59c8200f26f4f9ab5bce4184c7b0cb59, UNASSIGN in 1.4840 sec 2024-11-08T18:42:15,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-08T18:42:15,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4870 sec 2024-11-08T18:42:15,349 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091335349"}]},"ts":"1731091335349"} 2024-11-08T18:42:15,350 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T18:42:15,350 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T18:42:15,351 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4960 sec 2024-11-08T18:42:16,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-08T18:42:16,001 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:16,003 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T18:42:16,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:16,008 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:16,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T18:42:16,010 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:16,013 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:16,017 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/recovered.edits] 2024-11-08T18:42:16,021 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/0aff1cbbacbd4abd98d61f009aeaf993 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/0aff1cbbacbd4abd98d61f009aeaf993 2024-11-08T18:42:16,022 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/881a87b609ea41acb9ba71cc7c5bfccf to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/881a87b609ea41acb9ba71cc7c5bfccf 2024-11-08T18:42:16,024 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/ec5e48fd12eb4867831821680bb71d5c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/A/ec5e48fd12eb4867831821680bb71d5c 2024-11-08T18:42:16,025 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1a2c12ddfff3489db9f7ece7fd486d57 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/1a2c12ddfff3489db9f7ece7fd486d57 2024-11-08T18:42:16,026 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/912683af5a424fdb8a2cf61d0539b55e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/912683af5a424fdb8a2cf61d0539b55e 2024-11-08T18:42:16,027 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/cc25178018fb41f89543311d53a50535 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/B/cc25178018fb41f89543311d53a50535 2024-11-08T18:42:16,029 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/b92d1660de90444dbe329af9e54968c6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/b92d1660de90444dbe329af9e54968c6 2024-11-08T18:42:16,029 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/db35719ba3fb4a0c969abb13a05914eb to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/db35719ba3fb4a0c969abb13a05914eb 2024-11-08T18:42:16,030 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e405580bb6b3456f923b59ed5e3d8807 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/C/e405580bb6b3456f923b59ed5e3d8807 2024-11-08T18:42:16,031 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/recovered.edits/339.seqid to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59/recovered.edits/339.seqid 2024-11-08T18:42:16,032 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/59c8200f26f4f9ab5bce4184c7b0cb59 2024-11-08T18:42:16,032 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T18:42:16,033 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:16,035 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T18:42:16,037 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T18:42:16,038 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:16,038 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T18:42:16,038 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731091336038"}]},"ts":"9223372036854775807"} 2024-11-08T18:42:16,040 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T18:42:16,040 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 59c8200f26f4f9ab5bce4184c7b0cb59, NAME => 'TestAcidGuarantees,,1731091304676.59c8200f26f4f9ab5bce4184c7b0cb59.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T18:42:16,040 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T18:42:16,040 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731091336040"}]},"ts":"9223372036854775807"} 2024-11-08T18:42:16,041 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T18:42:16,041 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:16,042 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-08T18:42:16,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-08T18:42:16,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T18:42:16,123 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:16,132 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=238 (was 241), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=307 (was 353), ProcessCount=11 (was 11), AvailableMemoryMB=3715 (was 3728) 2024-11-08T18:42:16,143 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=307, ProcessCount=11, AvailableMemoryMB=3715 2024-11-08T18:42:16,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:42:16,144 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:42:16,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:16,146 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-08T18:42:16,146 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:16,146 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-08T18:42:16,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-08T18:42:16,147 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-08T18:42:16,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742327_1503 (size=959) 2024-11-08T18:42:16,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-08T18:42:16,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-08T18:42:16,558 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787 2024-11-08T18:42:16,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742328_1504 (size=53) 2024-11-08T18:42:16,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-08T18:42:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:42:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e98ed84368420c18855ecb5c81bb1538, disabling compactions & flushes 2024-11-08T18:42:16,968 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. after waiting 0 ms 2024-11-08T18:42:16,968 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:16,969 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:16,969 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e98ed84368420c18855ecb5c81bb1538: Waiting for close lock at 1731091336968Disabling compacts and flushes for region at 1731091336968Disabling writes for close at 1731091336968Writing region close event to WAL at 1731091336968Closed at 1731091336968 2024-11-08T18:42:16,972 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-08T18:42:16,972 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731091336972"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731091336972"}]},"ts":"1731091336972"} 2024-11-08T18:42:16,974 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-08T18:42:16,976 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-08T18:42:16,976 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091336976"}]},"ts":"1731091336976"} 2024-11-08T18:42:16,978 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-08T18:42:16,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, ASSIGN}] 2024-11-08T18:42:16,980 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, ASSIGN 2024-11-08T18:42:16,981 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, ASSIGN; state=OFFLINE, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=false 2024-11-08T18:42:17,132 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e98ed84368420c18855ecb5c81bb1538, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:42:17,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, ASSIGN because future has completed 2024-11-08T18:42:17,137 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:42:17,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-08T18:42:17,294 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:17,294 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:42:17,294 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,294 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:42:17,294 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,294 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,295 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,296 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e98ed84368420c18855ecb5c81bb1538 columnFamilyName A 2024-11-08T18:42:17,296 DEBUG [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:17,297 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(327): Store=e98ed84368420c18855ecb5c81bb1538/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:42:17,297 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,298 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e98ed84368420c18855ecb5c81bb1538 columnFamilyName B 2024-11-08T18:42:17,298 DEBUG [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:17,298 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(327): Store=e98ed84368420c18855ecb5c81bb1538/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:42:17,298 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,299 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e98ed84368420c18855ecb5c81bb1538 columnFamilyName C 2024-11-08T18:42:17,299 DEBUG [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:17,299 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(327): Store=e98ed84368420c18855ecb5c81bb1538/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:42:17,299 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,300 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,300 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,301 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,301 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,302 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:42:17,303 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,305 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-08T18:42:17,305 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened e98ed84368420c18855ecb5c81bb1538; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73573113, jitterRate=0.09632481634616852}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:42:17,305 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:17,306 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for e98ed84368420c18855ecb5c81bb1538: Running coprocessor pre-open hook at 1731091337294Writing region info on filesystem at 1731091337294Initializing all the Stores at 1731091337295 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091337295Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091337295Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091337295Cleaning up temporary data from old regions at 1731091337301 (+6 ms)Running coprocessor post-open hooks at 1731091337305 (+4 ms)Region opened successfully at 1731091337306 (+1 ms) 2024-11-08T18:42:17,307 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., pid=138, masterSystemTime=1731091337292 2024-11-08T18:42:17,308 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:17,308 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:17,309 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e98ed84368420c18855ecb5c81bb1538, regionState=OPEN, openSeqNum=2, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:42:17,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:42:17,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-08T18:42:17,313 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 in 174 msec 2024-11-08T18:42:17,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-08T18:42:17,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, ASSIGN in 335 msec 2024-11-08T18:42:17,316 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-08T18:42:17,316 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091337316"}]},"ts":"1731091337316"} 2024-11-08T18:42:17,317 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-08T18:42:17,318 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-08T18:42:17,320 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1740 sec 2024-11-08T18:42:18,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-08T18:42:18,301 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:18,304 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d463f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:18,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:18,305 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:18,307 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:18,307 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@330c90b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:18,311 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:18,312 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12e0701b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:18,312 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:18,314 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49420, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:18,315 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-08T18:42:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is b75fe890c219,42293,1731091182659 2024-11-08T18:42:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4eb8943a 2024-11-08T18:42:18,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-08T18:42:18,317 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-08T18:42:18,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-08T18:42:18,319 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-08T18:42:18,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:18,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742329_1505 (size=995) 2024-11-08T18:42:18,737 DEBUG [PEWorker-3 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-08T18:42:18,737 INFO [PEWorker-3 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-08T18:42:18,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:42:18,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, REOPEN/MOVE}] 2024-11-08T18:42:18,748 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, REOPEN/MOVE 2024-11-08T18:42:18,748 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e98ed84368420c18855ecb5c81bb1538, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:42:18,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, REOPEN/MOVE because future has completed 2024-11-08T18:42:18,750 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:42:18,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:42:18,903 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:18,903 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:42:18,903 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing e98ed84368420c18855ecb5c81bb1538, disabling compactions & flushes 2024-11-08T18:42:18,903 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:18,903 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:18,903 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. after waiting 0 ms 2024-11-08T18:42:18,904 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:18,944 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-08T18:42:18,946 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:18,946 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for e98ed84368420c18855ecb5c81bb1538: Waiting for close lock at 1731091338903Running coprocessor pre-close hooks at 1731091338903Disabling compacts and flushes for region at 1731091338903Disabling writes for close at 1731091338904 (+1 ms)Writing region close event to WAL at 1731091338905 (+1 ms)Running coprocessor post-close hooks at 1731091338946 (+41 ms)Closed at 1731091338946 2024-11-08T18:42:18,946 WARN [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: e98ed84368420c18855ecb5c81bb1538 to self. 2024-11-08T18:42:18,950 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:18,951 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e98ed84368420c18855ecb5c81bb1538, regionState=CLOSED 2024-11-08T18:42:18,953 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:42:18,958 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-08T18:42:18,959 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 in 205 msec 2024-11-08T18:42:18,959 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, REOPEN/MOVE; state=CLOSED, location=b75fe890c219,44447,1731091183487; forceNewPlan=false, retain=true 2024-11-08T18:42:19,110 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e98ed84368420c18855ecb5c81bb1538, regionState=OPENING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, REOPEN/MOVE because future has completed 2024-11-08T18:42:19,116 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:42:19,277 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,278 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} 2024-11-08T18:42:19,278 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,279 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-08T18:42:19,279 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,279 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,282 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,283 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e98ed84368420c18855ecb5c81bb1538 columnFamilyName A 2024-11-08T18:42:19,284 DEBUG [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:19,284 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(327): Store=e98ed84368420c18855ecb5c81bb1538/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:42:19,285 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,286 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e98ed84368420c18855ecb5c81bb1538 columnFamilyName B 2024-11-08T18:42:19,286 DEBUG [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:19,286 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(327): Store=e98ed84368420c18855ecb5c81bb1538/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:42:19,286 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,287 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e98ed84368420c18855ecb5c81bb1538 columnFamilyName C 2024-11-08T18:42:19,287 DEBUG [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:19,287 INFO [StoreOpener-e98ed84368420c18855ecb5c81bb1538-1 {}] regionserver.HStore(327): Store=e98ed84368420c18855ecb5c81bb1538/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-08T18:42:19,287 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,288 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,288 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,289 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,289 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,290 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-08T18:42:19,291 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,291 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened e98ed84368420c18855ecb5c81bb1538; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63895099, jitterRate=-0.04788883030414581}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-08T18:42:19,291 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,292 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for e98ed84368420c18855ecb5c81bb1538: Running coprocessor pre-open hook at 1731091339279Writing region info on filesystem at 1731091339279Initializing all the Stores at 1731091339281 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091339281Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091339281Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731091339281Cleaning up temporary data from old regions at 1731091339289 (+8 ms)Running coprocessor post-open hooks at 1731091339291 (+2 ms)Region opened successfully at 1731091339292 (+1 ms) 2024-11-08T18:42:19,292 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., pid=143, masterSystemTime=1731091339270 2024-11-08T18:42:19,294 DEBUG [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,294 INFO [RS_OPEN_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,294 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e98ed84368420c18855ecb5c81bb1538, regionState=OPEN, openSeqNum=5, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,295 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:42:19,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-08T18:42:19,297 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 in 181 msec 2024-11-08T18:42:19,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-08T18:42:19,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, REOPEN/MOVE in 550 msec 2024-11-08T18:42:19,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-08T18:42:19,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 555 msec 2024-11-08T18:42:19,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 981 msec 2024-11-08T18:42:19,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-08T18:42:19,303 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:19,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26378a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,304 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,304 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,304 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,304 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ece2cd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,305 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,306 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,306 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b590c67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,307 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1236062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,307 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,307 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,307 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@703cebf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,308 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,308 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40502, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,309 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@297684b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42a3e32a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,311 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f51645b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63449e34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,314 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@556fc022, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,314 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,314 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,314 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@121e869e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,316 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73139690, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,317 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77590f25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,317 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,318 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69e3ec55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,319 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40560, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,320 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267db557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13700528, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,321 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4712a1d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,321 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,322 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,322 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@735a02a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,323 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6deae39b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,323 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,323 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,323 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,323 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e743554, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,324 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d3f0e57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb7cff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,326 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,326 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50f2206, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,326 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,327 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,327 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ac192fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,328 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56ae597b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,328 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,328 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,329 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ade87a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,330 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39174ad3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request b75fe890c219,42293,-1 for getting cluster id 2024-11-08T18:42:19,331 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-08T18:42:19,331 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'dd42e0e3-0cec-490b-ade7-f4fb956445d9' 2024-11-08T18:42:19,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-08T18:42:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "dd42e0e3-0cec-490b-ade7-f4fb956445d9" 2024-11-08T18:42:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711659bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [b75fe890c219,42293,-1] 2024-11-08T18:42:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-08T18:42:19,332 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:19,333 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40654, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-08T18:42:19,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ac04d52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-08T18:42:19,334 DEBUG [Thread-2270 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,334 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,334 DEBUG [Thread-2276 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,334 DEBUG [Thread-2289 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,334 DEBUG [Thread-2272 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:19,334 DEBUG [Thread-2274 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,334 DEBUG [Thread-2285 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,334 DEBUG [Thread-2283 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,335 DEBUG [Thread-2281 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-08T18:42:19,335 DEBUG [Thread-2287 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T18:42:19,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49424, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49450, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,336 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49466, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,336 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,337 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:19,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49492, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,338 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,338 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=b75fe890c219,44447,1731091183487, seqNum=-1] 2024-11-08T18:42:19,338 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-08T18:42:19,338 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,339 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-08T18:42:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,340 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,341 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,342 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5] 2024-11-08T18:42:19,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,346 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:19,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-08T18:42:19,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110845a5fa70366447a4a6698a7b5cd7ea52_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091339345/Put/seqid=0 2024-11-08T18:42:19,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091399371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,376 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091399371, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742330_1506 (size=17034) 2024-11-08T18:42:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,377 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,377 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:19,380 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110845a5fa70366447a4a6698a7b5cd7ea52_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845a5fa70366447a4a6698a7b5cd7ea52_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091399375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,380 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,380 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091399376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091399376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,381 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,381 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f407a38afad04394b457c0ffa503e671, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:19,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f407a38afad04394b457c0ffa503e671 is 175, key is test_row_0/A:col10/1731091339345/Put/seqid=0 2024-11-08T18:42:19,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742331_1507 (size=48139) 2024-11-08T18:42:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T18:42:19,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091399479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091399479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,481 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:19,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:19,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091399490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,493 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091399490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,493 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091399490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,494 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,645 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T18:42:19,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091399689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,692 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091399689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,692 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091399699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091399699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:19,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091399699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:19,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:19,788 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f407a38afad04394b457c0ffa503e671 2024-11-08T18:42:19,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:19,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5f17ec721eb64e4cb67720a57ac8bda6 is 50, key is test_row_0/B:col10/1731091339345/Put/seqid=0 2024-11-08T18:42:19,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742332_1508 (size=12001) 2024-11-08T18:42:19,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:19,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T18:42:20,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091399999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091399999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091400010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091400010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091400010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,102 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:20,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:20,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5f17ec721eb64e4cb67720a57ac8bda6 2024-11-08T18:42:20,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/13400cca08b44ae8963f0afd1e3f6e7f is 50, key is test_row_0/C:col10/1731091339345/Put/seqid=0 2024-11-08T18:42:20,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742333_1509 (size=12001) 2024-11-08T18:42:20,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:20,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T18:42:20,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091400510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091400510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091400519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091400519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,521 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091400519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:20,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:20,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:20,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:20,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:20,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/13400cca08b44ae8963f0afd1e3f6e7f 2024-11-08T18:42:20,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f407a38afad04394b457c0ffa503e671 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f407a38afad04394b457c0ffa503e671 2024-11-08T18:42:20,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f407a38afad04394b457c0ffa503e671, entries=250, sequenceid=15, filesize=47.0 K 2024-11-08T18:42:20,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5f17ec721eb64e4cb67720a57ac8bda6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5f17ec721eb64e4cb67720a57ac8bda6 2024-11-08T18:42:20,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5f17ec721eb64e4cb67720a57ac8bda6, entries=150, sequenceid=15, filesize=11.7 K 2024-11-08T18:42:20,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/13400cca08b44ae8963f0afd1e3f6e7f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/13400cca08b44ae8963f0afd1e3f6e7f 2024-11-08T18:42:20,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/13400cca08b44ae8963f0afd1e3f6e7f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-08T18:42:20,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for e98ed84368420c18855ecb5c81bb1538 in 1303ms, sequenceid=15, compaction requested=false 2024-11-08T18:42:20,649 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-08T18:42:20,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:20,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-08T18:42:20,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:20,713 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T18:42:20,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082efb6b9962c24d888254df6d13bb5758_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091339374/Put/seqid=0 2024-11-08T18:42:20,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742334_1510 (size=12154) 2024-11-08T18:42:20,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:20,728 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082efb6b9962c24d888254df6d13bb5758_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082efb6b9962c24d888254df6d13bb5758_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f77c7973d09740e4a1b55e953a8b21ad, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:20,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f77c7973d09740e4a1b55e953a8b21ad is 175, key is test_row_0/A:col10/1731091339374/Put/seqid=0 2024-11-08T18:42:20,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742335_1511 (size=30955) 2024-11-08T18:42:20,862 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-08T18:42:21,139 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f77c7973d09740e4a1b55e953a8b21ad 2024-11-08T18:42:21,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/90b793e2d5e4452294b7ea45ecf16ade is 50, key is test_row_0/B:col10/1731091339374/Put/seqid=0 2024-11-08T18:42:21,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742336_1512 (size=12001) 2024-11-08T18:42:21,148 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/90b793e2d5e4452294b7ea45ecf16ade 2024-11-08T18:42:21,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/f17b418d36924cb0ae797ad7aa4ab478 is 50, key is test_row_0/C:col10/1731091339374/Put/seqid=0 2024-11-08T18:42:21,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742337_1513 (size=12001) 2024-11-08T18:42:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T18:42:21,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:21,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:21,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091401564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,570 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/f17b418d36924cb0ae797ad7aa4ab478 2024-11-08T18:42:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091401565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091401567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091401567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091401567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f77c7973d09740e4a1b55e953a8b21ad as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f77c7973d09740e4a1b55e953a8b21ad 2024-11-08T18:42:21,576 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f77c7973d09740e4a1b55e953a8b21ad, entries=150, sequenceid=41, filesize=30.2 K 2024-11-08T18:42:21,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/90b793e2d5e4452294b7ea45ecf16ade as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/90b793e2d5e4452294b7ea45ecf16ade 2024-11-08T18:42:21,580 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/90b793e2d5e4452294b7ea45ecf16ade, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T18:42:21,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/f17b418d36924cb0ae797ad7aa4ab478 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f17b418d36924cb0ae797ad7aa4ab478 2024-11-08T18:42:21,583 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f17b418d36924cb0ae797ad7aa4ab478, entries=150, sequenceid=41, filesize=11.7 K 2024-11-08T18:42:21,584 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=60.38 KB/61830 for e98ed84368420c18855ecb5c81bb1538 in 871ms, sequenceid=41, compaction requested=false 2024-11-08T18:42:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:21,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-08T18:42:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-08T18:42:21,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-08T18:42:21,586 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2470 sec 2024-11-08T18:42:21,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.2530 sec 2024-11-08T18:42:21,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:21,680 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:21,680 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-08T18:42:21,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110853b3eb5b9cf544ac8358ee83a1b1502e_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091341680/Put/seqid=0 2024-11-08T18:42:21,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742338_1514 (size=14594) 2024-11-08T18:42:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091401702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091401703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091401733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091401733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091401733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,738 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091401840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:21,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:21,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091401849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:21,851 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091402050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091402050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091402050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091402050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091402059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,101 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:22,104 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110853b3eb5b9cf544ac8358ee83a1b1502e_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110853b3eb5b9cf544ac8358ee83a1b1502e_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:22,104 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/512975a942e14a94877e77865eb5c80a, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:22,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/512975a942e14a94877e77865eb5c80a is 175, key is test_row_0/A:col10/1731091341680/Put/seqid=0 2024-11-08T18:42:22,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742339_1515 (size=39549) 2024-11-08T18:42:22,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091402359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,360 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091402359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091402359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091402360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,363 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091402369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,371 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,512 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/512975a942e14a94877e77865eb5c80a 2024-11-08T18:42:22,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/076febbd578045cd8bb4fd7f223d39b9 is 50, key is test_row_0/B:col10/1731091341680/Put/seqid=0 2024-11-08T18:42:22,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742340_1516 (size=12001) 2024-11-08T18:42:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091402869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091402869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091402869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091402870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:22,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091402879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:22,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/076febbd578045cd8bb4fd7f223d39b9 2024-11-08T18:42:22,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/b217e6b7e02840e5becd47e6d5a6b59a is 50, key is test_row_0/C:col10/1731091341680/Put/seqid=0 2024-11-08T18:42:22,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742341_1517 (size=12001) 2024-11-08T18:42:23,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/b217e6b7e02840e5becd47e6d5a6b59a 2024-11-08T18:42:23,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/512975a942e14a94877e77865eb5c80a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/512975a942e14a94877e77865eb5c80a 2024-11-08T18:42:23,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/512975a942e14a94877e77865eb5c80a, entries=200, sequenceid=55, filesize=38.6 K 2024-11-08T18:42:23,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/076febbd578045cd8bb4fd7f223d39b9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/076febbd578045cd8bb4fd7f223d39b9 2024-11-08T18:42:23,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/076febbd578045cd8bb4fd7f223d39b9, entries=150, sequenceid=55, filesize=11.7 K 2024-11-08T18:42:23,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/b217e6b7e02840e5becd47e6d5a6b59a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/b217e6b7e02840e5becd47e6d5a6b59a 2024-11-08T18:42:23,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/b217e6b7e02840e5becd47e6d5a6b59a, entries=150, sequenceid=55, filesize=11.7 K 2024-11-08T18:42:23,347 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=140.89 KB/144270 for e98ed84368420c18855ecb5c81bb1538 in 1667ms, sequenceid=55, compaction requested=true 2024-11-08T18:42:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:23,347 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:23,347 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:23,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:23,348 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:23,348 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/A is initiating minor compaction (all files) 2024-11-08T18:42:23,348 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:23,348 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/A in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:23,348 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/B is initiating minor compaction (all files) 2024-11-08T18:42:23,348 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/B in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:23,348 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f407a38afad04394b457c0ffa503e671, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f77c7973d09740e4a1b55e953a8b21ad, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/512975a942e14a94877e77865eb5c80a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=115.9 K 2024-11-08T18:42:23,348 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5f17ec721eb64e4cb67720a57ac8bda6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/90b793e2d5e4452294b7ea45ecf16ade, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/076febbd578045cd8bb4fd7f223d39b9] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=35.2 K 2024-11-08T18:42:23,348 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:23,348 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f407a38afad04394b457c0ffa503e671, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f77c7973d09740e4a1b55e953a8b21ad, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/512975a942e14a94877e77865eb5c80a] 2024-11-08T18:42:23,349 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f17ec721eb64e4cb67720a57ac8bda6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731091339339 2024-11-08T18:42:23,349 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f407a38afad04394b457c0ffa503e671, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731091339339 2024-11-08T18:42:23,349 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b793e2d5e4452294b7ea45ecf16ade, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731091339369 2024-11-08T18:42:23,349 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting f77c7973d09740e4a1b55e953a8b21ad, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731091339369 2024-11-08T18:42:23,349 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 076febbd578045cd8bb4fd7f223d39b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731091341564 2024-11-08T18:42:23,349 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 512975a942e14a94877e77865eb5c80a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731091341564 2024-11-08T18:42:23,355 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:23,355 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d762e6404b924407a2f7d79a51404366 is 50, key is test_row_0/B:col10/1731091341680/Put/seqid=0 2024-11-08T18:42:23,357 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:23,358 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411080760e63f7d744dbdbf391bb7c5529832_e98ed84368420c18855ecb5c81bb1538 store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:23,360 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411080760e63f7d744dbdbf391bb7c5529832_e98ed84368420c18855ecb5c81bb1538, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:23,360 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080760e63f7d744dbdbf391bb7c5529832_e98ed84368420c18855ecb5c81bb1538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:23,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742342_1518 (size=12104) 2024-11-08T18:42:23,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742343_1519 (size=4469) 2024-11-08T18:42:23,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-08T18:42:23,499 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:23,500 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:23,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-08T18:42:23,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T18:42:23,501 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:23,502 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:23,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:23,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T18:42:23,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-08T18:42:23,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:23,654 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-08T18:42:23,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108786ff26246eb4d60bbdef5430729f2bc_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091341708/Put/seqid=0 2024-11-08T18:42:23,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742344_1520 (size=12154) 2024-11-08T18:42:23,766 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d762e6404b924407a2f7d79a51404366 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d762e6404b924407a2f7d79a51404366 2024-11-08T18:42:23,769 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/B of e98ed84368420c18855ecb5c81bb1538 into d762e6404b924407a2f7d79a51404366(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:23,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:23,769 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/B, priority=13, startTime=1731091343347; duration=0sec 2024-11-08T18:42:23,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:23,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:B 2024-11-08T18:42:23,769 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:23,770 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:23,770 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/C is initiating minor compaction (all files) 2024-11-08T18:42:23,770 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/C in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:23,770 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/13400cca08b44ae8963f0afd1e3f6e7f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f17b418d36924cb0ae797ad7aa4ab478, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/b217e6b7e02840e5becd47e6d5a6b59a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=35.2 K 2024-11-08T18:42:23,770 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 13400cca08b44ae8963f0afd1e3f6e7f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731091339339 2024-11-08T18:42:23,770 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting f17b418d36924cb0ae797ad7aa4ab478, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731091339369 2024-11-08T18:42:23,770 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting b217e6b7e02840e5becd47e6d5a6b59a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731091341564 2024-11-08T18:42:23,775 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#C#compaction#441 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:23,776 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/5a9771f6ca6e4a25aa7cb7059d133d0c is 50, key is test_row_0/C:col10/1731091341680/Put/seqid=0 2024-11-08T18:42:23,777 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#A#compaction#439 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:23,777 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/05c9578e39f14fe886330e3d27175102 is 175, key is test_row_0/A:col10/1731091341680/Put/seqid=0 2024-11-08T18:42:23,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742345_1521 (size=12104) 2024-11-08T18:42:23,788 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/5a9771f6ca6e4a25aa7cb7059d133d0c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5a9771f6ca6e4a25aa7cb7059d133d0c 2024-11-08T18:42:23,792 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/C of e98ed84368420c18855ecb5c81bb1538 into 5a9771f6ca6e4a25aa7cb7059d133d0c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:23,792 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:23,792 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/C, priority=13, startTime=1731091343347; duration=0sec 2024-11-08T18:42:23,792 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:23,792 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:C 2024-11-08T18:42:23,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742346_1522 (size=31058) 2024-11-08T18:42:23,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T18:42:23,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:23,881 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:23,897 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091403894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:23,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091403895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:23,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091403896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:23,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:23,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091403896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:23,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091403899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:23,901 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in e98ed84368420c18855ecb5c81bb1538 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:42:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091404009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091404009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091404009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091404009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,011 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,104 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108786ff26246eb4d60bbdef5430729f2bc_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108786ff26246eb4d60bbdef5430729f2bc_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:24,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/028467a062da47c28f60c170252cbd4b, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:24,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/028467a062da47c28f60c170252cbd4b is 175, key is test_row_0/A:col10/1731091341708/Put/seqid=0 2024-11-08T18:42:24,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742347_1523 (size=30955) 2024-11-08T18:42:24,113 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/028467a062da47c28f60c170252cbd4b 2024-11-08T18:42:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5c001b158bc646aba88fcaa612d6053d is 50, key is test_row_0/B:col10/1731091341708/Put/seqid=0 2024-11-08T18:42:24,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742348_1524 (size=12001) 2024-11-08T18:42:24,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T18:42:24,203 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/05c9578e39f14fe886330e3d27175102 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/05c9578e39f14fe886330e3d27175102 2024-11-08T18:42:24,206 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/A of e98ed84368420c18855ecb5c81bb1538 into 05c9578e39f14fe886330e3d27175102(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:24,206 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:24,206 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/A, priority=13, startTime=1731091343347; duration=0sec 2024-11-08T18:42:24,206 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:24,206 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:A 2024-11-08T18:42:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091404219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091404219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091404219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091404219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,528 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5c001b158bc646aba88fcaa612d6053d 2024-11-08T18:42:24,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091404530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091404530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091404530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:24,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091404530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:24,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:24,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/c525900409244f5bb26f9485b09f9390 is 50, key is test_row_0/C:col10/1731091341708/Put/seqid=0 2024-11-08T18:42:24,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742349_1525 (size=12001) 2024-11-08T18:42:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T18:42:24,937 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/c525900409244f5bb26f9485b09f9390 2024-11-08T18:42:24,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/028467a062da47c28f60c170252cbd4b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/028467a062da47c28f60c170252cbd4b 2024-11-08T18:42:24,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/028467a062da47c28f60c170252cbd4b, entries=150, sequenceid=79, filesize=30.2 K 2024-11-08T18:42:24,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5c001b158bc646aba88fcaa612d6053d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c001b158bc646aba88fcaa612d6053d 2024-11-08T18:42:24,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,946 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c001b158bc646aba88fcaa612d6053d, entries=150, sequenceid=79, filesize=11.7 K 2024-11-08T18:42:24,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/c525900409244f5bb26f9485b09f9390 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c525900409244f5bb26f9485b09f9390 2024-11-08T18:42:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,952 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c525900409244f5bb26f9485b09f9390, entries=150, sequenceid=79, filesize=11.7 K 2024-11-08T18:42:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,952 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for e98ed84368420c18855ecb5c81bb1538 in 1298ms, sequenceid=79, compaction requested=false 2024-11-08T18:42:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:24,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:24,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-08T18:42:24,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-08T18:42:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-08T18:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4520 sec 2024-11-08T18:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.4560 sec 2024-11-08T18:42:24,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:24,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:25,041 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:25,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-08T18:42:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c32d0adb1b5e41d9ac4804e8851129b3_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_1/A:col10/1731091345041/Put/seqid=0 2024-11-08T18:42:25,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742350_1526 (size=14594) 2024-11-08T18:42:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091405066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,070 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091405067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,071 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091405069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091405070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091405179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091405179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091405179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,181 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091405179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,181 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091405389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091405389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091405389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091405389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,393 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,458 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:25,461 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108c32d0adb1b5e41d9ac4804e8851129b3_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c32d0adb1b5e41d9ac4804e8851129b3_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:25,461 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/07bf2267c918490c844526c0588d1a18, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:25,462 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/07bf2267c918490c844526c0588d1a18 is 175, key is test_row_1/A:col10/1731091345041/Put/seqid=0 2024-11-08T18:42:25,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742351_1527 (size=39545) 2024-11-08T18:42:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-08T18:42:25,649 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:25,650 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-08T18:42:25,651 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T18:42:25,652 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:25,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091405700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091405700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091405700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091405700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,704 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T18:42:25,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T18:42:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:25,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:25,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:25,865 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/07bf2267c918490c844526c0588d1a18 2024-11-08T18:42:25,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/078c3e97ab3c42508982e3e7eb0300b4 is 50, key is test_row_1/B:col10/1731091345041/Put/seqid=0 2024-11-08T18:42:25,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742352_1528 (size=9657) 2024-11-08T18:42:25,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091405919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:25,920 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in e98ed84368420c18855ecb5c81bb1538 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:42:25,920 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:25,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:25,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T18:42:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:25,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:25,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T18:42:26,109 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T18:42:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,109 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:26,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:26,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091406209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091406209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091406209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:26,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091406209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:26,213 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:26,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T18:42:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:26,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:26,274 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/078c3e97ab3c42508982e3e7eb0300b4 2024-11-08T18:42:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T18:42:26,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/94fbed45b5ff4b8f8fff168ade50e193 is 50, key is test_row_1/C:col10/1731091345041/Put/seqid=0 2024-11-08T18:42:26,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742353_1529 (size=9657) 2024-11-08T18:42:26,288 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/94fbed45b5ff4b8f8fff168ade50e193 2024-11-08T18:42:26,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/07bf2267c918490c844526c0588d1a18 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/07bf2267c918490c844526c0588d1a18 2024-11-08T18:42:26,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/07bf2267c918490c844526c0588d1a18, entries=200, sequenceid=97, filesize=38.6 K 2024-11-08T18:42:26,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/078c3e97ab3c42508982e3e7eb0300b4 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/078c3e97ab3c42508982e3e7eb0300b4 2024-11-08T18:42:26,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/078c3e97ab3c42508982e3e7eb0300b4, entries=100, sequenceid=97, filesize=9.4 K 2024-11-08T18:42:26,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/94fbed45b5ff4b8f8fff168ade50e193 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/94fbed45b5ff4b8f8fff168ade50e193 2024-11-08T18:42:26,300 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/94fbed45b5ff4b8f8fff168ade50e193, entries=100, sequenceid=97, filesize=9.4 K 2024-11-08T18:42:26,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for e98ed84368420c18855ecb5c81bb1538 in 1260ms, sequenceid=97, compaction requested=true 2024-11-08T18:42:26,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:26,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:26,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:26,302 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:26,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:26,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:26,302 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:26,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:26,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:26,302 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101558 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:26,302 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:26,302 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/B is initiating minor compaction (all files) 2024-11-08T18:42:26,302 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/A is initiating minor compaction (all files) 2024-11-08T18:42:26,302 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/A in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,302 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/B in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,302 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d762e6404b924407a2f7d79a51404366, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c001b158bc646aba88fcaa612d6053d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/078c3e97ab3c42508982e3e7eb0300b4] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=33.0 K 2024-11-08T18:42:26,302 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/05c9578e39f14fe886330e3d27175102, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/028467a062da47c28f60c170252cbd4b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/07bf2267c918490c844526c0588d1a18] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=99.2 K 2024-11-08T18:42:26,302 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,303 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/05c9578e39f14fe886330e3d27175102, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/028467a062da47c28f60c170252cbd4b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/07bf2267c918490c844526c0588d1a18] 2024-11-08T18:42:26,305 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d762e6404b924407a2f7d79a51404366, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731091341564 2024-11-08T18:42:26,305 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05c9578e39f14fe886330e3d27175102, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731091341564 2024-11-08T18:42:26,305 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 028467a062da47c28f60c170252cbd4b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731091341701 2024-11-08T18:42:26,305 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c001b158bc646aba88fcaa612d6053d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731091341701 2024-11-08T18:42:26,305 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07bf2267c918490c844526c0588d1a18, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731091343885 2024-11-08T18:42:26,305 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 078c3e97ab3c42508982e3e7eb0300b4, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731091345040 2024-11-08T18:42:26,310 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:26,312 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411080a9b1286609445769f91db30258c8dd2_e98ed84368420c18855ecb5c81bb1538 store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:26,313 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#B#compaction#448 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:26,313 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411080a9b1286609445769f91db30258c8dd2_e98ed84368420c18855ecb5c81bb1538, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:26,313 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411080a9b1286609445769f91db30258c8dd2_e98ed84368420c18855ecb5c81bb1538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:26,313 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/3074005c29a94539a7c68b155e44e813 is 50, key is test_row_0/B:col10/1731091341708/Put/seqid=0 2024-11-08T18:42:26,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742355_1531 (size=4469) 2024-11-08T18:42:26,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742354_1530 (size=12207) 2024-11-08T18:42:26,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-08T18:42:26,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,414 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T18:42:26,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085b4edc2e61fe48e9a2e8eaa0cb62fe66_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091345065/Put/seqid=0 2024-11-08T18:42:26,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742356_1532 (size=12154) 2024-11-08T18:42:26,718 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#A#compaction#447 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:26,718 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/3bfc6eafee8e4eaf8af357f0e45a6ce2 is 175, key is test_row_0/A:col10/1731091341708/Put/seqid=0 2024-11-08T18:42:26,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742357_1533 (size=31268) 2024-11-08T18:42:26,725 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/3074005c29a94539a7c68b155e44e813 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3074005c29a94539a7c68b155e44e813 2024-11-08T18:42:26,728 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/B of e98ed84368420c18855ecb5c81bb1538 into 3074005c29a94539a7c68b155e44e813(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:26,728 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:26,728 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/B, priority=13, startTime=1731091346302; duration=0sec 2024-11-08T18:42:26,728 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:26,728 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:B 2024-11-08T18:42:26,728 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:26,729 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:26,729 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/C is initiating minor compaction (all files) 2024-11-08T18:42:26,729 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/C in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:26,729 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5a9771f6ca6e4a25aa7cb7059d133d0c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c525900409244f5bb26f9485b09f9390, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/94fbed45b5ff4b8f8fff168ade50e193] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=33.0 K 2024-11-08T18:42:26,730 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a9771f6ca6e4a25aa7cb7059d133d0c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731091341564 2024-11-08T18:42:26,730 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting c525900409244f5bb26f9485b09f9390, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1731091341701 2024-11-08T18:42:26,730 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 94fbed45b5ff4b8f8fff168ade50e193, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731091345040 2024-11-08T18:42:26,736 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#C#compaction#450 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:26,737 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/8acfade59d0d4086876599b8f8df19d7 is 50, key is test_row_0/C:col10/1731091341708/Put/seqid=0 2024-11-08T18:42:26,744 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742358_1534 (size=12207) 2024-11-08T18:42:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T18:42:26,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:26,830 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085b4edc2e61fe48e9a2e8eaa0cb62fe66_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b4edc2e61fe48e9a2e8eaa0cb62fe66_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:26,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/128565ca948b4969aa351d45e002af5f, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:26,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/128565ca948b4969aa351d45e002af5f is 175, key is test_row_0/A:col10/1731091345065/Put/seqid=0 2024-11-08T18:42:26,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742359_1535 (size=30955) 2024-11-08T18:42:27,124 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/3bfc6eafee8e4eaf8af357f0e45a6ce2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/3bfc6eafee8e4eaf8af357f0e45a6ce2 2024-11-08T18:42:27,127 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/A of e98ed84368420c18855ecb5c81bb1538 into 3bfc6eafee8e4eaf8af357f0e45a6ce2(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:27,127 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:27,127 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/A, priority=13, startTime=1731091346301; duration=0sec 2024-11-08T18:42:27,127 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:27,127 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:A 2024-11-08T18:42:27,147 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/8acfade59d0d4086876599b8f8df19d7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8acfade59d0d4086876599b8f8df19d7 2024-11-08T18:42:27,150 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/C of e98ed84368420c18855ecb5c81bb1538 into 8acfade59d0d4086876599b8f8df19d7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:27,150 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:27,150 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/C, priority=13, startTime=1731091346302; duration=0sec 2024-11-08T18:42:27,150 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:27,150 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:C 2024-11-08T18:42:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:27,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:27,234 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091407230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,234 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/128565ca948b4969aa351d45e002af5f 2024-11-08T18:42:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091407231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091407231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091407232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,235 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5c70aaa8118e44619e7ce9038a396ffb is 50, key is test_row_0/B:col10/1731091345065/Put/seqid=0 2024-11-08T18:42:27,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742360_1536 (size=12001) 2024-11-08T18:42:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091407339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091407339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091407339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091407339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,342 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091407549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091407549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091407549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091407549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,550 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,551 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,643 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5c70aaa8118e44619e7ce9038a396ffb 2024-11-08T18:42:27,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/3eb734bdf2e740769ed35d27cbca277a is 50, key is test_row_0/C:col10/1731091345065/Put/seqid=0 2024-11-08T18:42:27,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742361_1537 (size=12001) 2024-11-08T18:42:27,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T18:42:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091407859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091407860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091407860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091407860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:27,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,059 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/3eb734bdf2e740769ed35d27cbca277a 2024-11-08T18:42:28,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/128565ca948b4969aa351d45e002af5f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/128565ca948b4969aa351d45e002af5f 2024-11-08T18:42:28,065 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/128565ca948b4969aa351d45e002af5f, entries=150, sequenceid=118, filesize=30.2 K 2024-11-08T18:42:28,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5c70aaa8118e44619e7ce9038a396ffb as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c70aaa8118e44619e7ce9038a396ffb 2024-11-08T18:42:28,068 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c70aaa8118e44619e7ce9038a396ffb, entries=150, sequenceid=118, filesize=11.7 K 2024-11-08T18:42:28,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/3eb734bdf2e740769ed35d27cbca277a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/3eb734bdf2e740769ed35d27cbca277a 2024-11-08T18:42:28,070 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/3eb734bdf2e740769ed35d27cbca277a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-08T18:42:28,071 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for e98ed84368420c18855ecb5c81bb1538 in 1657ms, sequenceid=118, compaction requested=false 2024-11-08T18:42:28,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:28,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:28,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-08T18:42:28,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-08T18:42:28,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-08T18:42:28,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4200 sec 2024-11-08T18:42:28,075 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 2.4240 sec 2024-11-08T18:42:28,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:28,373 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:28,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T18:42:28,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110817b408f3338048e28442d771cfd7b9db_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091348373/Put/seqid=0 2024-11-08T18:42:28,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742362_1538 (size=14794) 2024-11-08T18:42:28,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091408390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091408391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,398 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091408394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091408396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091408499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,501 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091408499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091408509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091408509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091408710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091408710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091408719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091408719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:28,721 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:28,785 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:28,788 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110817b408f3338048e28442d771cfd7b9db_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110817b408f3338048e28442d771cfd7b9db_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:28,788 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/a34dd484f1874898b3fbc904ed05140e, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:28,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/a34dd484f1874898b3fbc904ed05140e is 175, key is test_row_0/A:col10/1731091348373/Put/seqid=0 2024-11-08T18:42:28,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742363_1539 (size=39749) 2024-11-08T18:42:29,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091409019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091409019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:29,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091409029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091409030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:29,193 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=139, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/a34dd484f1874898b3fbc904ed05140e 2024-11-08T18:42:29,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/8995ae42bd73498fbc17da15e5d553db is 50, key is test_row_0/B:col10/1731091348373/Put/seqid=0 2024-11-08T18:42:29,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742364_1540 (size=12151) 2024-11-08T18:42:29,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091409529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091409529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:29,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091409539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:29,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091409539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,541 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,542 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:29,630 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/8995ae42bd73498fbc17da15e5d553db 2024-11-08T18:42:29,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/cf98c8307b444bdfb80665b65f59aca6 is 50, key is test_row_0/C:col10/1731091348373/Put/seqid=0 2024-11-08T18:42:29,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742365_1541 (size=12151) 2024-11-08T18:42:29,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-08T18:42:29,810 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:29,810 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-08T18:42:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T18:42:29,811 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:29,812 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:29,812 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:29,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T18:42:29,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T18:42:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:29,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:29,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:29,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:29,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:29,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49492 deadline: 1731091409969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:29,970 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to b75fe890c219,44447,1731091183487 for 'test_row_0' in e98ed84368420c18855ecb5c81bb1538 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-08T18:42:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:29,971 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,038 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=139 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/cf98c8307b444bdfb80665b65f59aca6 2024-11-08T18:42:30,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/a34dd484f1874898b3fbc904ed05140e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/a34dd484f1874898b3fbc904ed05140e 2024-11-08T18:42:30,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/a34dd484f1874898b3fbc904ed05140e, entries=200, sequenceid=139, filesize=38.8 K 2024-11-08T18:42:30,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/8995ae42bd73498fbc17da15e5d553db as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/8995ae42bd73498fbc17da15e5d553db 2024-11-08T18:42:30,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/8995ae42bd73498fbc17da15e5d553db, entries=150, sequenceid=139, filesize=11.9 K 2024-11-08T18:42:30,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/cf98c8307b444bdfb80665b65f59aca6 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/cf98c8307b444bdfb80665b65f59aca6 2024-11-08T18:42:30,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/cf98c8307b444bdfb80665b65f59aca6, entries=150, sequenceid=139, filesize=11.9 K 2024-11-08T18:42:30,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e98ed84368420c18855ecb5c81bb1538 in 1677ms, sequenceid=139, compaction requested=true 2024-11-08T18:42:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:30,050 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:30,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:30,050 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:30,051 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:30,051 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101972 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:30,051 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/B is initiating minor compaction (all files) 2024-11-08T18:42:30,051 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/A is initiating minor compaction (all files) 2024-11-08T18:42:30,051 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/B in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:30,051 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/A in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:30,051 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3074005c29a94539a7c68b155e44e813, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c70aaa8118e44619e7ce9038a396ffb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/8995ae42bd73498fbc17da15e5d553db] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=35.5 K 2024-11-08T18:42:30,051 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/3bfc6eafee8e4eaf8af357f0e45a6ce2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/128565ca948b4969aa351d45e002af5f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/a34dd484f1874898b3fbc904ed05140e] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=99.6 K 2024-11-08T18:42:30,051 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:30,051 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/3bfc6eafee8e4eaf8af357f0e45a6ce2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/128565ca948b4969aa351d45e002af5f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/a34dd484f1874898b3fbc904ed05140e] 2024-11-08T18:42:30,051 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3074005c29a94539a7c68b155e44e813, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731091341708 2024-11-08T18:42:30,052 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bfc6eafee8e4eaf8af357f0e45a6ce2, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731091341708 2024-11-08T18:42:30,052 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c70aaa8118e44619e7ce9038a396ffb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731091345064 2024-11-08T18:42:30,052 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8995ae42bd73498fbc17da15e5d553db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731091347229 2024-11-08T18:42:30,052 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 128565ca948b4969aa351d45e002af5f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731091345064 2024-11-08T18:42:30,052 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting a34dd484f1874898b3fbc904ed05140e, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731091347229 2024-11-08T18:42:30,056 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:30,057 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411085d7e53746b62434a81020e582075ef6d_e98ed84368420c18855ecb5c81bb1538 store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:30,057 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#B#compaction#456 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:30,057 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/4a9518ffdf4d46ab82997986dedaeefc is 50, key is test_row_0/B:col10/1731091348373/Put/seqid=0 2024-11-08T18:42:30,059 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411085d7e53746b62434a81020e582075ef6d_e98ed84368420c18855ecb5c81bb1538, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:30,059 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411085d7e53746b62434a81020e582075ef6d_e98ed84368420c18855ecb5c81bb1538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:30,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742366_1542 (size=12459) 2024-11-08T18:42:30,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742367_1543 (size=4469) 2024-11-08T18:42:30,066 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#A#compaction#457 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:30,067 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/e9744cacfc324e4da5c1a861ec740c23 is 175, key is test_row_0/A:col10/1731091348373/Put/seqid=0 2024-11-08T18:42:30,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742368_1544 (size=31413) 2024-11-08T18:42:30,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-08T18:42:30,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:30,117 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-08T18:42:30,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082253b64b7de9408fb6ebe0a8f509f730_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091348395/Put/seqid=0 2024-11-08T18:42:30,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742369_1545 (size=12304) 2024-11-08T18:42:30,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T18:42:30,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T18:42:30,466 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/4a9518ffdf4d46ab82997986dedaeefc as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/4a9518ffdf4d46ab82997986dedaeefc 2024-11-08T18:42:30,469 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/B of e98ed84368420c18855ecb5c81bb1538 into 4a9518ffdf4d46ab82997986dedaeefc(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:30,469 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:30,469 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/B, priority=13, startTime=1731091350050; duration=0sec 2024-11-08T18:42:30,469 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:30,469 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:B 2024-11-08T18:42:30,470 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:30,470 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36359 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:30,470 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/C is initiating minor compaction (all files) 2024-11-08T18:42:30,471 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/C in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:30,471 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8acfade59d0d4086876599b8f8df19d7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/3eb734bdf2e740769ed35d27cbca277a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/cf98c8307b444bdfb80665b65f59aca6] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=35.5 K 2024-11-08T18:42:30,471 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8acfade59d0d4086876599b8f8df19d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731091341708 2024-11-08T18:42:30,471 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3eb734bdf2e740769ed35d27cbca277a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731091345064 2024-11-08T18:42:30,471 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting cf98c8307b444bdfb80665b65f59aca6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731091347229 2024-11-08T18:42:30,473 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/e9744cacfc324e4da5c1a861ec740c23 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/e9744cacfc324e4da5c1a861ec740c23 2024-11-08T18:42:30,476 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/A of e98ed84368420c18855ecb5c81bb1538 into e9744cacfc324e4da5c1a861ec740c23(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:30,476 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:30,476 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#C#compaction#459 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:30,476 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/A, priority=13, startTime=1731091350050; duration=0sec 2024-11-08T18:42:30,477 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:30,477 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:A 2024-11-08T18:42:30,477 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/020aa6b4204a4ccabac6facdd0aa5df8 is 50, key is test_row_0/C:col10/1731091348373/Put/seqid=0 2024-11-08T18:42:30,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742370_1546 (size=12459) 2024-11-08T18:42:30,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:30,529 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411082253b64b7de9408fb6ebe0a8f509f730_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082253b64b7de9408fb6ebe0a8f509f730_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:30,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/442eff680547472d8ad78f72622469c2, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:30,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/442eff680547472d8ad78f72622469c2 is 175, key is test_row_0/A:col10/1731091348395/Put/seqid=0 2024-11-08T18:42:30,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742371_1547 (size=31105) 2024-11-08T18:42:30,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:30,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:30,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091410560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091410563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091410564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091410565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,569 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,671 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091410669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,671 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091410680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091410680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091410680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091410879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,884 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/020aa6b4204a4ccabac6facdd0aa5df8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/020aa6b4204a4ccabac6facdd0aa5df8 2024-11-08T18:42:30,887 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/C of e98ed84368420c18855ecb5c81bb1538 into 020aa6b4204a4ccabac6facdd0aa5df8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:30,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:30,887 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/C, priority=13, startTime=1731091350050; duration=0sec 2024-11-08T18:42:30,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:30,887 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:C 2024-11-08T18:42:30,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091410890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091410890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091410890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:30,893 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:30,933 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/442eff680547472d8ad78f72622469c2 2024-11-08T18:42:30,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5333576dade64244951e665332e9b16f is 50, key is test_row_0/B:col10/1731091348395/Put/seqid=0 2024-11-08T18:42:30,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742372_1548 (size=12151) 2024-11-08T18:42:30,943 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5333576dade64244951e665332e9b16f 2024-11-08T18:42:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T18:42:30,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/c622436585cb47139d3fafff59f89eea is 50, key is test_row_0/C:col10/1731091348395/Put/seqid=0 2024-11-08T18:42:30,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742373_1549 (size=12151) 2024-11-08T18:42:31,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091411189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091411199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091411199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,201 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091411199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,357 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/c622436585cb47139d3fafff59f89eea 2024-11-08T18:42:31,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/442eff680547472d8ad78f72622469c2 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/442eff680547472d8ad78f72622469c2 2024-11-08T18:42:31,363 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/442eff680547472d8ad78f72622469c2, entries=150, sequenceid=157, filesize=30.4 K 2024-11-08T18:42:31,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/5333576dade64244951e665332e9b16f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5333576dade64244951e665332e9b16f 2024-11-08T18:42:31,365 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5333576dade64244951e665332e9b16f, entries=150, sequenceid=157, filesize=11.9 K 2024-11-08T18:42:31,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/c622436585cb47139d3fafff59f89eea as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c622436585cb47139d3fafff59f89eea 2024-11-08T18:42:31,368 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c622436585cb47139d3fafff59f89eea, entries=150, sequenceid=157, filesize=11.9 K 2024-11-08T18:42:31,369 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for e98ed84368420c18855ecb5c81bb1538 in 1252ms, sequenceid=157, compaction requested=false 2024-11-08T18:42:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:31,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-08T18:42:31,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-08T18:42:31,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-08T18:42:31,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5580 sec 2024-11-08T18:42:31,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.5620 sec 2024-11-08T18:42:31,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:31,702 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:31,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-08T18:42:31,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cdfa3e7d437842f1927fee0bea8d5420_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091350564/Put/seqid=0 2024-11-08T18:42:31,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742374_1550 (size=12304) 2024-11-08T18:42:31,712 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:31,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108cdfa3e7d437842f1927fee0bea8d5420_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cdfa3e7d437842f1927fee0bea8d5420_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:31,715 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/738de1da19fa49d0a2b9b8258db9f193, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:31,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/738de1da19fa49d0a2b9b8258db9f193 is 175, key is test_row_0/A:col10/1731091350564/Put/seqid=0 2024-11-08T18:42:31,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742375_1551 (size=31105) 2024-11-08T18:42:31,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091411723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091411726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091411727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091411727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091411829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091411839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,842 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091411839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091411840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:31,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:31,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-08T18:42:31,960 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:31,960 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:31,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-08T18:42:31,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T18:42:31,962 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:31,962 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:31,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:32,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091412039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,041 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091412049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091412050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091412050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T18:42:32,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T18:42:32,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:32,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,120 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/738de1da19fa49d0a2b9b8258db9f193 2024-11-08T18:42:32,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/62ff0269fc914b309c7d9ddd6cf6096e is 50, key is test_row_0/B:col10/1731091350564/Put/seqid=0 2024-11-08T18:42:32,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742376_1552 (size=12151) 2024-11-08T18:42:32,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T18:42:32,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:32,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T18:42:32,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091412349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,352 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091412359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091412359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,362 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091412359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T18:42:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/62ff0269fc914b309c7d9ddd6cf6096e 2024-11-08T18:42:32,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/7eef1a45f791416b83ab0664191f0567 is 50, key is test_row_0/C:col10/1731091350564/Put/seqid=0 2024-11-08T18:42:32,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742377_1553 (size=12151) 2024-11-08T18:42:32,572 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T18:42:32,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:32,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,572 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T18:42:32,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T18:42:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,725 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091412859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091412869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091412869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091412869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:32,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:32,877 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T18:42:32,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:32,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:32,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/7eef1a45f791416b83ab0664191f0567 2024-11-08T18:42:32,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/738de1da19fa49d0a2b9b8258db9f193 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/738de1da19fa49d0a2b9b8258db9f193 2024-11-08T18:42:32,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/738de1da19fa49d0a2b9b8258db9f193, entries=150, sequenceid=179, filesize=30.4 K 2024-11-08T18:42:32,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/62ff0269fc914b309c7d9ddd6cf6096e as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/62ff0269fc914b309c7d9ddd6cf6096e 2024-11-08T18:42:32,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/62ff0269fc914b309c7d9ddd6cf6096e, entries=150, sequenceid=179, filesize=11.9 K 2024-11-08T18:42:32,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/7eef1a45f791416b83ab0664191f0567 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/7eef1a45f791416b83ab0664191f0567 2024-11-08T18:42:32,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/7eef1a45f791416b83ab0664191f0567, entries=150, sequenceid=179, filesize=11.9 K 2024-11-08T18:42:32,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for e98ed84368420c18855ecb5c81bb1538 in 1247ms, sequenceid=179, compaction requested=true 2024-11-08T18:42:32,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:32,950 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:32,950 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:32,950 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:32,950 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:32,950 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/B is initiating minor compaction (all files) 2024-11-08T18:42:32,950 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/B in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,951 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/4a9518ffdf4d46ab82997986dedaeefc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5333576dade64244951e665332e9b16f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/62ff0269fc914b309c7d9ddd6cf6096e] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=35.9 K 2024-11-08T18:42:32,951 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93623 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:32,951 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/A is initiating minor compaction (all files) 2024-11-08T18:42:32,951 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/A in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,951 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/e9744cacfc324e4da5c1a861ec740c23, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/442eff680547472d8ad78f72622469c2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/738de1da19fa49d0a2b9b8258db9f193] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=91.4 K 2024-11-08T18:42:32,951 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a9518ffdf4d46ab82997986dedaeefc, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731091347229 2024-11-08T18:42:32,951 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,951 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/e9744cacfc324e4da5c1a861ec740c23, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/442eff680547472d8ad78f72622469c2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/738de1da19fa49d0a2b9b8258db9f193] 2024-11-08T18:42:32,951 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5333576dade64244951e665332e9b16f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731091348388 2024-11-08T18:42:32,951 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9744cacfc324e4da5c1a861ec740c23, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731091347229 2024-11-08T18:42:32,951 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 62ff0269fc914b309c7d9ddd6cf6096e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731091350563 2024-11-08T18:42:32,952 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 442eff680547472d8ad78f72622469c2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731091348388 2024-11-08T18:42:32,952 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 738de1da19fa49d0a2b9b8258db9f193, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731091350563 2024-11-08T18:42:32,956 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:32,957 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#B#compaction#465 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:32,958 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d76ffee2fc1d4a31a004c5b963f4f6b4 is 50, key is test_row_0/B:col10/1731091350564/Put/seqid=0 2024-11-08T18:42:32,959 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411088651c9957c9242519f864a4a0f2137a2_e98ed84368420c18855ecb5c81bb1538 store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:32,961 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411088651c9957c9242519f864a4a0f2137a2_e98ed84368420c18855ecb5c81bb1538, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:32,961 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411088651c9957c9242519f864a4a0f2137a2_e98ed84368420c18855ecb5c81bb1538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:32,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742378_1554 (size=12561) 2024-11-08T18:42:32,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742379_1555 (size=4469) 2024-11-08T18:42:32,966 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#A#compaction#466 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:32,966 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/70f636107fdc412292ab9cbb4d0401b5 is 175, key is test_row_0/A:col10/1731091350564/Put/seqid=0 2024-11-08T18:42:32,968 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d76ffee2fc1d4a31a004c5b963f4f6b4 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d76ffee2fc1d4a31a004c5b963f4f6b4 2024-11-08T18:42:32,971 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/B of e98ed84368420c18855ecb5c81bb1538 into d76ffee2fc1d4a31a004c5b963f4f6b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:32,971 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:32,971 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/B, priority=13, startTime=1731091352950; duration=0sec 2024-11-08T18:42:32,972 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:32,972 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:B 2024-11-08T18:42:32,972 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:32,972 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:32,972 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/C is initiating minor compaction (all files) 2024-11-08T18:42:32,972 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/C in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:32,973 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/020aa6b4204a4ccabac6facdd0aa5df8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c622436585cb47139d3fafff59f89eea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/7eef1a45f791416b83ab0664191f0567] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=35.9 K 2024-11-08T18:42:32,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742380_1556 (size=31515) 2024-11-08T18:42:32,974 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 020aa6b4204a4ccabac6facdd0aa5df8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=139, earliestPutTs=1731091347229 2024-11-08T18:42:32,974 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting c622436585cb47139d3fafff59f89eea, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731091348388 2024-11-08T18:42:32,974 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eef1a45f791416b83ab0664191f0567, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731091350563 2024-11-08T18:42:32,977 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/70f636107fdc412292ab9cbb4d0401b5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/70f636107fdc412292ab9cbb4d0401b5 2024-11-08T18:42:32,980 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#C#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:32,980 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/8b2a7c91ff7b4cc4b5cc3eeaaf17877f is 50, key is test_row_0/C:col10/1731091350564/Put/seqid=0 2024-11-08T18:42:32,981 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/A of e98ed84368420c18855ecb5c81bb1538 into 70f636107fdc412292ab9cbb4d0401b5(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:32,981 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:32,981 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/A, priority=13, startTime=1731091352949; duration=0sec 2024-11-08T18:42:32,981 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:32,981 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:A 2024-11-08T18:42:32,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742381_1557 (size=12561) 2024-11-08T18:42:33,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-08T18:42:33,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:33,030 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-08T18:42:33,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110829efab9a189b44d481b8c893c419ecd0_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091351726/Put/seqid=0 2024-11-08T18:42:33,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742382_1558 (size=12304) 2024-11-08T18:42:33,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T18:42:33,387 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/8b2a7c91ff7b4cc4b5cc3eeaaf17877f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8b2a7c91ff7b4cc4b5cc3eeaaf17877f 2024-11-08T18:42:33,390 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/C of e98ed84368420c18855ecb5c81bb1538 into 8b2a7c91ff7b4cc4b5cc3eeaaf17877f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:33,390 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:33,390 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/C, priority=13, startTime=1731091352950; duration=0sec 2024-11-08T18:42:33,390 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:33,390 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:C 2024-11-08T18:42:33,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:33,442 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110829efab9a189b44d481b8c893c419ecd0_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110829efab9a189b44d481b8c893c419ecd0_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:33,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/18a1da38ef4041a98589bf2a2b667e3b, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:33,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/18a1da38ef4041a98589bf2a2b667e3b is 175, key is test_row_0/A:col10/1731091351726/Put/seqid=0 2024-11-08T18:42:33,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742383_1559 (size=31105) 2024-11-08T18:42:33,846 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=198, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/18a1da38ef4041a98589bf2a2b667e3b 2024-11-08T18:42:33,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d4a4d6e95bbe41a783c27fa2d4874c54 is 50, key is test_row_0/B:col10/1731091351726/Put/seqid=0 2024-11-08T18:42:33,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742384_1560 (size=12151) 2024-11-08T18:42:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:33,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:33,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091413893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:33,897 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:33,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091413897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:33,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:33,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091413897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091413898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:33,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091414010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091414010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091414010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091414010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T18:42:34,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091414219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091414219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,221 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091414219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091414219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,256 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d4a4d6e95bbe41a783c27fa2d4874c54 2024-11-08T18:42:34,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/81d2795fd7c7415292459881099a71e3 is 50, key is test_row_0/C:col10/1731091351726/Put/seqid=0 2024-11-08T18:42:34,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742385_1561 (size=12151) 2024-11-08T18:42:34,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091414529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,531 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,531 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091414529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091414530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091414530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:34,696 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=198 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/81d2795fd7c7415292459881099a71e3 2024-11-08T18:42:34,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/18a1da38ef4041a98589bf2a2b667e3b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/18a1da38ef4041a98589bf2a2b667e3b 2024-11-08T18:42:34,702 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/18a1da38ef4041a98589bf2a2b667e3b, entries=150, sequenceid=198, filesize=30.4 K 2024-11-08T18:42:34,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d4a4d6e95bbe41a783c27fa2d4874c54 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d4a4d6e95bbe41a783c27fa2d4874c54 2024-11-08T18:42:34,705 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d4a4d6e95bbe41a783c27fa2d4874c54, entries=150, sequenceid=198, filesize=11.9 K 2024-11-08T18:42:34,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/81d2795fd7c7415292459881099a71e3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/81d2795fd7c7415292459881099a71e3 2024-11-08T18:42:34,708 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/81d2795fd7c7415292459881099a71e3, entries=150, sequenceid=198, filesize=11.9 K 2024-11-08T18:42:34,709 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for e98ed84368420c18855ecb5c81bb1538 in 1679ms, sequenceid=198, compaction requested=false 2024-11-08T18:42:34,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:34,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:34,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-08T18:42:34,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-08T18:42:34,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-08T18:42:34,712 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7480 sec 2024-11-08T18:42:34,713 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.7520 sec 2024-11-08T18:42:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:35,042 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:35,042 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-08T18:42:35,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081650f9f8696340459d92e9aebc92f68b_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091355041/Put/seqid=0 2024-11-08T18:42:35,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742386_1562 (size=12304) 2024-11-08T18:42:35,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091415080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,084 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091415081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091415082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091415083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,085 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091415189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091415189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,191 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091415189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091415189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,192 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091415400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091415400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091415400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,402 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091415400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,480 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:35,483 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081650f9f8696340459d92e9aebc92f68b_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081650f9f8696340459d92e9aebc92f68b_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:35,483 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/b1406523d9a54fd4bd459c79595eb42a, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:35,484 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/b1406523d9a54fd4bd459c79595eb42a is 175, key is test_row_0/A:col10/1731091355041/Put/seqid=0 2024-11-08T18:42:35,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742387_1563 (size=31105) 2024-11-08T18:42:35,492 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/b1406523d9a54fd4bd459c79595eb42a 2024-11-08T18:42:35,498 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/3d809364ad154b5eb77f66c47dc7186c is 50, key is test_row_0/B:col10/1731091355041/Put/seqid=0 2024-11-08T18:42:35,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742388_1564 (size=12151) 2024-11-08T18:42:35,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091415709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091415709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091415710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091415710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:35,712 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:35,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/3d809364ad154b5eb77f66c47dc7186c 2024-11-08T18:42:35,910 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/5bccac82ef7249dfa76144cee6f570d1 is 50, key is test_row_0/C:col10/1731091355041/Put/seqid=0 2024-11-08T18:42:35,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742389_1565 (size=12151) 2024-11-08T18:42:36,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-08T18:42:36,129 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:36,129 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-08T18:42:36,131 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:36,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T18:42:36,131 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:36,132 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:36,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091416220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:36,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091416220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:36,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091416220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:36,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:36,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091416220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:36,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T18:42:36,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T18:42:36,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:36,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:36,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:36,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:36,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:36,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:36,313 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/5bccac82ef7249dfa76144cee6f570d1 2024-11-08T18:42:36,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/b1406523d9a54fd4bd459c79595eb42a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/b1406523d9a54fd4bd459c79595eb42a 2024-11-08T18:42:36,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/b1406523d9a54fd4bd459c79595eb42a, entries=150, sequenceid=223, filesize=30.4 K 2024-11-08T18:42:36,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/3d809364ad154b5eb77f66c47dc7186c as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3d809364ad154b5eb77f66c47dc7186c 2024-11-08T18:42:36,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3d809364ad154b5eb77f66c47dc7186c, entries=150, sequenceid=223, filesize=11.9 K 2024-11-08T18:42:36,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/5bccac82ef7249dfa76144cee6f570d1 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5bccac82ef7249dfa76144cee6f570d1 2024-11-08T18:42:36,325 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5bccac82ef7249dfa76144cee6f570d1, entries=150, sequenceid=223, filesize=11.9 K 2024-11-08T18:42:36,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for e98ed84368420c18855ecb5c81bb1538 in 1283ms, sequenceid=223, compaction requested=true 2024-11-08T18:42:36,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:36,326 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:36,326 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:36,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:36,326 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93725 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:36,326 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:36,326 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/B is initiating minor compaction (all files) 2024-11-08T18:42:36,326 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/A is initiating minor compaction (all files) 2024-11-08T18:42:36,326 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/A in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:36,327 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/B in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:36,327 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/70f636107fdc412292ab9cbb4d0401b5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/18a1da38ef4041a98589bf2a2b667e3b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/b1406523d9a54fd4bd459c79595eb42a] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=91.5 K 2024-11-08T18:42:36,327 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d76ffee2fc1d4a31a004c5b963f4f6b4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d4a4d6e95bbe41a783c27fa2d4874c54, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3d809364ad154b5eb77f66c47dc7186c] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=36.0 K 2024-11-08T18:42:36,327 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:36,327 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/70f636107fdc412292ab9cbb4d0401b5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/18a1da38ef4041a98589bf2a2b667e3b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/b1406523d9a54fd4bd459c79595eb42a] 2024-11-08T18:42:36,327 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70f636107fdc412292ab9cbb4d0401b5, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731091350563 2024-11-08T18:42:36,327 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d76ffee2fc1d4a31a004c5b963f4f6b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731091350563 2024-11-08T18:42:36,327 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting d4a4d6e95bbe41a783c27fa2d4874c54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731091351725 2024-11-08T18:42:36,327 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18a1da38ef4041a98589bf2a2b667e3b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731091351725 2024-11-08T18:42:36,327 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d809364ad154b5eb77f66c47dc7186c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731091353895 2024-11-08T18:42:36,328 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1406523d9a54fd4bd459c79595eb42a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731091353895 2024-11-08T18:42:36,332 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:36,333 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411081120bae7ba70462b98d18bcaf5bb2b5e_e98ed84368420c18855ecb5c81bb1538 store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:36,334 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:36,334 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/13b7ad79c7024d90babde102ddc79268 is 50, key is test_row_0/B:col10/1731091355041/Put/seqid=0 2024-11-08T18:42:36,335 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411081120bae7ba70462b98d18bcaf5bb2b5e_e98ed84368420c18855ecb5c81bb1538, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:36,335 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081120bae7ba70462b98d18bcaf5bb2b5e_e98ed84368420c18855ecb5c81bb1538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:36,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742390_1566 (size=12663) 2024-11-08T18:42:36,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742391_1567 (size=4469) 2024-11-08T18:42:36,340 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#A#compaction#474 average throughput is 3.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:36,340 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/4d55c21ca6e649319df244484b99514f is 175, key is test_row_0/A:col10/1731091355041/Put/seqid=0 2024-11-08T18:42:36,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742392_1568 (size=31617) 2024-11-08T18:42:36,436 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-08T18:42:36,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:36,436 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-08T18:42:36,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a758e2a67c5e43f8aaa8c8e0d8eb9d48_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091355081/Put/seqid=0 2024-11-08T18:42:36,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742393_1569 (size=12304) 2024-11-08T18:42:36,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T18:42:36,741 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/13b7ad79c7024d90babde102ddc79268 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/13b7ad79c7024d90babde102ddc79268 2024-11-08T18:42:36,745 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/B of e98ed84368420c18855ecb5c81bb1538 into 13b7ad79c7024d90babde102ddc79268(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:36,745 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:36,745 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/B, priority=13, startTime=1731091356326; duration=0sec 2024-11-08T18:42:36,745 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:36,745 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:B 2024-11-08T18:42:36,745 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:36,746 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/4d55c21ca6e649319df244484b99514f as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/4d55c21ca6e649319df244484b99514f 2024-11-08T18:42:36,746 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:36,746 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/C is initiating minor compaction (all files) 2024-11-08T18:42:36,746 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/C in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:36,746 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8b2a7c91ff7b4cc4b5cc3eeaaf17877f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/81d2795fd7c7415292459881099a71e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5bccac82ef7249dfa76144cee6f570d1] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=36.0 K 2024-11-08T18:42:36,747 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b2a7c91ff7b4cc4b5cc3eeaaf17877f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731091350563 2024-11-08T18:42:36,747 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 81d2795fd7c7415292459881099a71e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=198, earliestPutTs=1731091351725 2024-11-08T18:42:36,747 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bccac82ef7249dfa76144cee6f570d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731091353895 2024-11-08T18:42:36,750 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/A of e98ed84368420c18855ecb5c81bb1538 into 4d55c21ca6e649319df244484b99514f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:36,750 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:36,750 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/A, priority=13, startTime=1731091356325; duration=0sec 2024-11-08T18:42:36,750 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:36,750 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:A 2024-11-08T18:42:36,753 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#C#compaction#477 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:36,754 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/63491121dbf3460d80c64f9e022b4ef5 is 50, key is test_row_0/C:col10/1731091355041/Put/seqid=0 2024-11-08T18:42:36,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742394_1570 (size=12663) 2024-11-08T18:42:36,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T18:42:36,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:36,848 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108a758e2a67c5e43f8aaa8c8e0d8eb9d48_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a758e2a67c5e43f8aaa8c8e0d8eb9d48_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:36,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/94a51a413aac4a8198fba37de9a118b9, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:36,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/94a51a413aac4a8198fba37de9a118b9 is 175, key is test_row_0/A:col10/1731091355081/Put/seqid=0 2024-11-08T18:42:36,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742395_1571 (size=31105) 2024-11-08T18:42:37,159 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/63491121dbf3460d80c64f9e022b4ef5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/63491121dbf3460d80c64f9e022b4ef5 2024-11-08T18:42:37,162 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/C of e98ed84368420c18855ecb5c81bb1538 into 63491121dbf3460d80c64f9e022b4ef5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:37,162 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:37,162 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/C, priority=13, startTime=1731091356326; duration=0sec 2024-11-08T18:42:37,163 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:37,163 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:C 2024-11-08T18:42:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:37,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:37,252 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/94a51a413aac4a8198fba37de9a118b9 2024-11-08T18:42:37,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/3f81075a36a845a1a305a24bdce07cb8 is 50, key is test_row_0/B:col10/1731091355081/Put/seqid=0 2024-11-08T18:42:37,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742396_1572 (size=12151) 2024-11-08T18:42:37,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091417257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091417259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,263 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091417261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091417261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,264 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T18:42:37,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091417370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091417370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091417370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091417370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,373 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091417579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091417579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091417580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091417580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,582 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,583 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,661 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/3f81075a36a845a1a305a24bdce07cb8 2024-11-08T18:42:37,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/a6ea6b9cb65b478ca19fd85b6c1d1066 is 50, key is test_row_0/C:col10/1731091355081/Put/seqid=0 2024-11-08T18:42:37,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742397_1573 (size=12151) 2024-11-08T18:42:37,675 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/a6ea6b9cb65b478ca19fd85b6c1d1066 2024-11-08T18:42:37,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/94a51a413aac4a8198fba37de9a118b9 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/94a51a413aac4a8198fba37de9a118b9 2024-11-08T18:42:37,681 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/94a51a413aac4a8198fba37de9a118b9, entries=150, sequenceid=235, filesize=30.4 K 2024-11-08T18:42:37,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/3f81075a36a845a1a305a24bdce07cb8 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3f81075a36a845a1a305a24bdce07cb8 2024-11-08T18:42:37,684 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3f81075a36a845a1a305a24bdce07cb8, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T18:42:37,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/a6ea6b9cb65b478ca19fd85b6c1d1066 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/a6ea6b9cb65b478ca19fd85b6c1d1066 2024-11-08T18:42:37,687 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/a6ea6b9cb65b478ca19fd85b6c1d1066, entries=150, sequenceid=235, filesize=11.9 K 2024-11-08T18:42:37,688 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for e98ed84368420c18855ecb5c81bb1538 in 1252ms, sequenceid=235, compaction requested=false 2024-11-08T18:42:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:37,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-08T18:42:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-08T18:42:37,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-08T18:42:37,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5580 sec 2024-11-08T18:42:37,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 1.5620 sec 2024-11-08T18:42:37,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:37,891 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-08T18:42:37,891 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-08T18:42:37,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110823a76e8cde694777bd09ac11aa849e21_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091357260/Put/seqid=0 2024-11-08T18:42:37,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091417895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091417897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,901 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091417897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,902 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091417898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:37,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742398_1574 (size=14944) 2024-11-08T18:42:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091418009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091418009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091418009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091418009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091418219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091418219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091418219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091418219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-08T18:42:38,279 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:38,279 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-08T18:42:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-08T18:42:38,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T18:42:38,281 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-08T18:42:38,282 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-08T18:42:38,282 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-08T18:42:38,305 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:38,308 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110823a76e8cde694777bd09ac11aa849e21_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110823a76e8cde694777bd09ac11aa849e21_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:38,308 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/fa82bead8e8e4bb898aa05cb31fb77cf, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:38,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/fa82bead8e8e4bb898aa05cb31fb77cf is 175, key is test_row_0/A:col10/1731091357260/Put/seqid=0 2024-11-08T18:42:38,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742399_1575 (size=39899) 2024-11-08T18:42:38,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T18:42:38,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:38,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091418530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091418530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091418530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091418530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,533 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:38,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:38,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:38,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T18:42:38,712 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/fa82bead8e8e4bb898aa05cb31fb77cf 2024-11-08T18:42:38,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/89b67f9edf1b4d0984cb64e56feb237d is 50, key is test_row_0/B:col10/1731091357260/Put/seqid=0 2024-11-08T18:42:38,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742400_1576 (size=12251) 2024-11-08T18:42:38,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:38,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:38,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:38,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:38,739 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:38,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:38,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:38,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:38,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:38,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T18:42:39,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49424 deadline: 1731091419040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:39,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49476 deadline: 1731091419040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:39,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:39,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:39,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49450 deadline: 1731091419040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:39,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-08T18:42:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:39,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49474 deadline: 1731091419040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 2024-11-08T18:42:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 , the old value is region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-08T18:42:39,042 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., hostname=b75fe890c219,44447,1731091183487, seqNum=5 because the exception is null or not the one we care about 2024-11-08T18:42:39,044 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,044 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,120 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/89b67f9edf1b4d0984cb64e56feb237d 2024-11-08T18:42:39,125 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/64e2bfb7d3804328a3c9c55f810c3328 is 50, key is test_row_0/C:col10/1731091357260/Put/seqid=0 2024-11-08T18:42:39,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742401_1577 (size=12251) 2024-11-08T18:42:39,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:39,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,336 INFO [Thread-2285 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2285. 2024-11-08T18:42:39,336 DEBUG [Thread-2285 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:39,336 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,336 DEBUG [Thread-2285 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:39,337 INFO [Thread-2281 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2281. 2024-11-08T18:42:39,337 DEBUG [Thread-2281 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:39,337 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,337 DEBUG [Thread-2281 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:39,338 INFO [Thread-2289 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2289. 2024-11-08T18:42:39,339 DEBUG [Thread-2289 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:39,339 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,339 DEBUG [Thread-2289 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,339 INFO [Thread-2287 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2287. 2024-11-08T18:42:39,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:39,339 DEBUG [Thread-2287 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:39,339 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,339 DEBUG [Thread-2287 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:39,340 INFO [Thread-2283 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2283. 2024-11-08T18:42:39,340 DEBUG [Thread-2283 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:39,340 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,340 DEBUG [Thread-2283 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:39,340 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:39,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T18:42:39,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:39,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-08T18:42:39,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/64e2bfb7d3804328a3c9c55f810c3328 2024-11-08T18:42:39,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/fa82bead8e8e4bb898aa05cb31fb77cf as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/fa82bead8e8e4bb898aa05cb31fb77cf 2024-11-08T18:42:39,543 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/fa82bead8e8e4bb898aa05cb31fb77cf, entries=200, sequenceid=265, filesize=39.0 K 2024-11-08T18:42:39,544 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/89b67f9edf1b4d0984cb64e56feb237d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/89b67f9edf1b4d0984cb64e56feb237d 2024-11-08T18:42:39,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/89b67f9edf1b4d0984cb64e56feb237d, entries=150, sequenceid=265, filesize=12.0 K 2024-11-08T18:42:39,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/64e2bfb7d3804328a3c9c55f810c3328 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/64e2bfb7d3804328a3c9c55f810c3328 2024-11-08T18:42:39,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/64e2bfb7d3804328a3c9c55f810c3328, entries=150, sequenceid=265, filesize=12.0 K 2024-11-08T18:42:39,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for e98ed84368420c18855ecb5c81bb1538 in 1665ms, sequenceid=265, compaction requested=true 2024-11-08T18:42:39,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:39,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:A, priority=-2147483648, current under compaction store size is 1 2024-11-08T18:42:39,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:39,557 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:39,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:B, priority=-2147483648, current under compaction store size is 2 2024-11-08T18:42:39,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:39,557 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:39,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e98ed84368420c18855ecb5c81bb1538:C, priority=-2147483648, current under compaction store size is 3 2024-11-08T18:42:39,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:39,558 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:39,558 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102621 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:39,558 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/A is initiating minor compaction (all files) 2024-11-08T18:42:39,558 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/B is initiating minor compaction (all files) 2024-11-08T18:42:39,558 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/A in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,558 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/B in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,558 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/13b7ad79c7024d90babde102ddc79268, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3f81075a36a845a1a305a24bdce07cb8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/89b67f9edf1b4d0984cb64e56feb237d] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=36.2 K 2024-11-08T18:42:39,558 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/4d55c21ca6e649319df244484b99514f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/94a51a413aac4a8198fba37de9a118b9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/fa82bead8e8e4bb898aa05cb31fb77cf] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=100.2 K 2024-11-08T18:42:39,559 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,559 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. files: [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/4d55c21ca6e649319df244484b99514f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/94a51a413aac4a8198fba37de9a118b9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/fa82bead8e8e4bb898aa05cb31fb77cf] 2024-11-08T18:42:39,559 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 13b7ad79c7024d90babde102ddc79268, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731091353895 2024-11-08T18:42:39,559 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d55c21ca6e649319df244484b99514f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731091353895 2024-11-08T18:42:39,559 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f81075a36a845a1a305a24bdce07cb8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731091355080 2024-11-08T18:42:39,560 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting 94a51a413aac4a8198fba37de9a118b9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731091355080 2024-11-08T18:42:39,560 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 89b67f9edf1b4d0984cb64e56feb237d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731091357252 2024-11-08T18:42:39,560 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa82bead8e8e4bb898aa05cb31fb77cf, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731091357252 2024-11-08T18:42:39,567 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:39,569 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#B#compaction#483 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:39,569 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241108d11ef186c3034e13a4b17ee81163d6bd_e98ed84368420c18855ecb5c81bb1538 store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:39,569 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/13a6f878d1734ba0a993c937cf304e12 is 50, key is test_row_0/B:col10/1731091357260/Put/seqid=0 2024-11-08T18:42:39,578 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241108d11ef186c3034e13a4b17ee81163d6bd_e98ed84368420c18855ecb5c81bb1538, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:39,578 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241108d11ef186c3034e13a4b17ee81163d6bd_e98ed84368420c18855ecb5c81bb1538 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:39,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742402_1578 (size=12865) 2024-11-08T18:42:39,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742403_1579 (size=4469) 2024-11-08T18:42:39,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44447 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-08T18:42:39,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,657 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-08T18:42:39,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110892adfab4fcaa4fefb15bbdc84c04f0b1_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091357896/Put/seqid=0 2024-11-08T18:42:39,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742404_1580 (size=12454) 2024-11-08T18:42:39,983 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#A#compaction#484 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:39,984 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/882d83836ddc4667ab34007e28719c0b is 175, key is test_row_0/A:col10/1731091357260/Put/seqid=0 2024-11-08T18:42:39,989 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/13a6f878d1734ba0a993c937cf304e12 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/13a6f878d1734ba0a993c937cf304e12 2024-11-08T18:42:39,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742405_1581 (size=31819) 2024-11-08T18:42:39,995 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/B of e98ed84368420c18855ecb5c81bb1538 into 13a6f878d1734ba0a993c937cf304e12(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:39,995 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:39,995 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/B, priority=13, startTime=1731091359557; duration=0sec 2024-11-08T18:42:39,995 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-08T18:42:39,995 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:B 2024-11-08T18:42:39,995 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-08T18:42:39,996 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37065 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-08T18:42:39,996 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1541): e98ed84368420c18855ecb5c81bb1538/C is initiating minor compaction (all files) 2024-11-08T18:42:39,996 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e98ed84368420c18855ecb5c81bb1538/C in TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:39,996 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/63491121dbf3460d80c64f9e022b4ef5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/a6ea6b9cb65b478ca19fd85b6c1d1066, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/64e2bfb7d3804328a3c9c55f810c3328] into tmpdir=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp, totalSize=36.2 K 2024-11-08T18:42:39,997 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 63491121dbf3460d80c64f9e022b4ef5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731091353895 2024-11-08T18:42:39,997 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting a6ea6b9cb65b478ca19fd85b6c1d1066, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1731091355080 2024-11-08T18:42:39,997 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e2bfb7d3804328a3c9c55f810c3328, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731091357252 2024-11-08T18:42:40,004 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e98ed84368420c18855ecb5c81bb1538#C#compaction#486 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-08T18:42:40,004 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/e56a2b43f05f47288bf8336b131db938 is 50, key is test_row_0/C:col10/1731091357260/Put/seqid=0 2024-11-08T18:42:40,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742406_1582 (size=12865) 2024-11-08T18:42:40,022 INFO [Thread-2272 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2272. 2024-11-08T18:42:40,022 DEBUG [Thread-2272 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:40,022 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,022 DEBUG [Thread-2272 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,022 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:40,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. as already flushing 2024-11-08T18:42:40,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44447 {}] regionserver.HRegion(8855): Flush requested on e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:40,052 INFO [Thread-2270 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2270. 2024-11-08T18:42:40,052 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-11-08T18:42:40,052 INFO [Thread-2276 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2276. 2024-11-08T18:42:40,052 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:40,052 DEBUG [Thread-2270 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:40,052 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,052 DEBUG [Thread-2276 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:40,052 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,052 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,052 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,052 DEBUG [Thread-2270 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,052 DEBUG [Thread-2276 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:40,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:40,052 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:40,053 INFO [Thread-2274 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2274. 2024-11-08T18:42:40,053 DEBUG [Thread-2274 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-08T18:42:40,053 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,054 DEBUG [Thread-2274 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:40,054 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:40,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:40,072 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110892adfab4fcaa4fefb15bbdc84c04f0b1_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110892adfab4fcaa4fefb15bbdc84c04f0b1_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:40,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/5c2132cd602c4554b213268225ff9849, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:40,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/5c2132cd602c4554b213268225ff9849 is 175, key is test_row_0/A:col10/1731091357896/Put/seqid=0 2024-11-08T18:42:40,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742407_1583 (size=31255) 2024-11-08T18:42:40,400 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/882d83836ddc4667ab34007e28719c0b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/882d83836ddc4667ab34007e28719c0b 2024-11-08T18:42:40,405 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/A of e98ed84368420c18855ecb5c81bb1538 into 882d83836ddc4667ab34007e28719c0b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:40,405 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:40,406 INFO [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/A, priority=13, startTime=1731091359557; duration=0sec 2024-11-08T18:42:40,406 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:40,406 DEBUG [RS:0;b75fe890c219:44447-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:A 2024-11-08T18:42:40,414 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/e56a2b43f05f47288bf8336b131db938 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/e56a2b43f05f47288bf8336b131db938 2024-11-08T18:42:40,420 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e98ed84368420c18855ecb5c81bb1538/C of e98ed84368420c18855ecb5c81bb1538 into e56a2b43f05f47288bf8336b131db938(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-08T18:42:40,420 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:40,420 INFO [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., storeName=e98ed84368420c18855ecb5c81bb1538/C, priority=13, startTime=1731091359557; duration=0sec 2024-11-08T18:42:40,420 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-08T18:42:40,420 DEBUG [RS:0;b75fe890c219:44447-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e98ed84368420c18855ecb5c81bb1538:C 2024-11-08T18:42:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T18:42:40,481 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/5c2132cd602c4554b213268225ff9849 2024-11-08T18:42:40,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/74aeaaaf23c841d8991aee9523a34a48 is 50, key is test_row_0/B:col10/1731091357896/Put/seqid=0 2024-11-08T18:42:40,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742408_1584 (size=12301) 2024-11-08T18:42:40,898 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/74aeaaaf23c841d8991aee9523a34a48 2024-11-08T18:42:40,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/f1ba81ea61bf4f7a93d2b27765bd2d62 is 50, key is test_row_0/C:col10/1731091357896/Put/seqid=0 2024-11-08T18:42:40,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742409_1585 (size=12301) 2024-11-08T18:42:41,316 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/f1ba81ea61bf4f7a93d2b27765bd2d62 2024-11-08T18:42:41,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/5c2132cd602c4554b213268225ff9849 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/5c2132cd602c4554b213268225ff9849 2024-11-08T18:42:41,329 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/5c2132cd602c4554b213268225ff9849, entries=150, sequenceid=274, filesize=30.5 K 2024-11-08T18:42:41,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/74aeaaaf23c841d8991aee9523a34a48 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/74aeaaaf23c841d8991aee9523a34a48 2024-11-08T18:42:41,335 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/74aeaaaf23c841d8991aee9523a34a48, entries=150, sequenceid=274, filesize=12.0 K 2024-11-08T18:42:41,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/f1ba81ea61bf4f7a93d2b27765bd2d62 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f1ba81ea61bf4f7a93d2b27765bd2d62 2024-11-08T18:42:41,342 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f1ba81ea61bf4f7a93d2b27765bd2d62, entries=150, sequenceid=274, filesize=12.0 K 2024-11-08T18:42:41,343 INFO [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=33.54 KB/34350 for e98ed84368420c18855ecb5c81bb1538 in 1686ms, sequenceid=274, compaction requested=false 2024-11-08T18:42:41,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for e98ed84368420c18855ecb5c81bb1538: 2024-11-08T18:42:41,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:41,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/b75fe890c219:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-08T18:42:41,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-08T18:42:41,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-08T18:42:41,346 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0620 sec 2024-11-08T18:42:41,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 3.0680 sec 2024-11-08T18:42:41,639 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-08T18:42:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-08T18:42:42,441 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:42,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-08T18:42:42,441 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4154 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4065 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4135 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4147 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4062 2024-11-08T18:42:42,442 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-08T18:42:42,442 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T18:42:42,443 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:42:42,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:42,443 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:42,443 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:42,445 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-08T18:42:42,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:42,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T18:42:42,453 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091362453"}]},"ts":"1731091362453"} 2024-11-08T18:42:42,455 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-08T18:42:42,455 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-08T18:42:42,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-08T18:42:42,458 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, UNASSIGN}] 2024-11-08T18:42:42,459 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, UNASSIGN 2024-11-08T18:42:42,460 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=e98ed84368420c18855ecb5c81bb1538, regionState=CLOSING, regionLocation=b75fe890c219,44447,1731091183487 2024-11-08T18:42:42,461 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, UNASSIGN because future has completed 2024-11-08T18:42:42,462 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-08T18:42:42,462 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487}] 2024-11-08T18:42:42,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T18:42:42,616 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(122): Close e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:42,616 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-08T18:42:42,616 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1722): Closing e98ed84368420c18855ecb5c81bb1538, disabling compactions & flushes 2024-11-08T18:42:42,616 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:42,616 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:42,617 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. after waiting 0 ms 2024-11-08T18:42:42,617 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:42,617 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(2902): Flushing e98ed84368420c18855ecb5c81bb1538 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-08T18:42:42,627 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081d0345b1e26249aaa67d960b599eec24_e98ed84368420c18855ecb5c81bb1538 is 50, key is test_row_0/A:col10/1731091360050/Put/seqid=0 2024-11-08T18:42:42,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742410_1586 (size=12454) 2024-11-08T18:42:42,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T18:42:43,033 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-08T18:42:43,041 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411081d0345b1e26249aaa67d960b599eec24_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081d0345b1e26249aaa67d960b599eec24_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:43,042 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f1482f8a217d49818c773096430a8b5b, store: [table=TestAcidGuarantees family=A region=e98ed84368420c18855ecb5c81bb1538] 2024-11-08T18:42:43,043 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f1482f8a217d49818c773096430a8b5b is 175, key is test_row_0/A:col10/1731091360050/Put/seqid=0 2024-11-08T18:42:43,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742411_1587 (size=31255) 2024-11-08T18:42:43,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T18:42:43,451 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f1482f8a217d49818c773096430a8b5b 2024-11-08T18:42:43,464 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d578b2f2824f4210a79b6c6cb01f47a5 is 50, key is test_row_0/B:col10/1731091360050/Put/seqid=0 2024-11-08T18:42:43,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742412_1588 (size=12301) 2024-11-08T18:42:43,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T18:42:43,870 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d578b2f2824f4210a79b6c6cb01f47a5 2024-11-08T18:42:43,881 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/4489934285fd49f69df03f215863583b is 50, key is test_row_0/C:col10/1731091360050/Put/seqid=0 2024-11-08T18:42:43,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742413_1589 (size=12301) 2024-11-08T18:42:44,286 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/4489934285fd49f69df03f215863583b 2024-11-08T18:42:44,330 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/A/f1482f8a217d49818c773096430a8b5b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f1482f8a217d49818c773096430a8b5b 2024-11-08T18:42:44,334 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f1482f8a217d49818c773096430a8b5b, entries=150, sequenceid=285, filesize=30.5 K 2024-11-08T18:42:44,335 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/B/d578b2f2824f4210a79b6c6cb01f47a5 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d578b2f2824f4210a79b6c6cb01f47a5 2024-11-08T18:42:44,339 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d578b2f2824f4210a79b6c6cb01f47a5, entries=150, sequenceid=285, filesize=12.0 K 2024-11-08T18:42:44,340 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/.tmp/C/4489934285fd49f69df03f215863583b as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/4489934285fd49f69df03f215863583b 2024-11-08T18:42:44,343 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/4489934285fd49f69df03f215863583b, entries=150, sequenceid=285, filesize=12.0 K 2024-11-08T18:42:44,344 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e98ed84368420c18855ecb5c81bb1538 in 1726ms, sequenceid=285, compaction requested=true 2024-11-08T18:42:44,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f407a38afad04394b457c0ffa503e671, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f77c7973d09740e4a1b55e953a8b21ad, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/512975a942e14a94877e77865eb5c80a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/05c9578e39f14fe886330e3d27175102, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/028467a062da47c28f60c170252cbd4b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/07bf2267c918490c844526c0588d1a18, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/3bfc6eafee8e4eaf8af357f0e45a6ce2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/128565ca948b4969aa351d45e002af5f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/a34dd484f1874898b3fbc904ed05140e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/e9744cacfc324e4da5c1a861ec740c23, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/442eff680547472d8ad78f72622469c2, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/70f636107fdc412292ab9cbb4d0401b5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/738de1da19fa49d0a2b9b8258db9f193, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/18a1da38ef4041a98589bf2a2b667e3b, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/4d55c21ca6e649319df244484b99514f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/b1406523d9a54fd4bd459c79595eb42a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/94a51a413aac4a8198fba37de9a118b9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/fa82bead8e8e4bb898aa05cb31fb77cf] to archive 2024-11-08T18:42:44,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:42:44,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f407a38afad04394b457c0ffa503e671 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f407a38afad04394b457c0ffa503e671 2024-11-08T18:42:44,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f77c7973d09740e4a1b55e953a8b21ad to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f77c7973d09740e4a1b55e953a8b21ad 2024-11-08T18:42:44,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/512975a942e14a94877e77865eb5c80a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/512975a942e14a94877e77865eb5c80a 2024-11-08T18:42:44,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/05c9578e39f14fe886330e3d27175102 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/05c9578e39f14fe886330e3d27175102 2024-11-08T18:42:44,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/028467a062da47c28f60c170252cbd4b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/028467a062da47c28f60c170252cbd4b 2024-11-08T18:42:44,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/07bf2267c918490c844526c0588d1a18 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/07bf2267c918490c844526c0588d1a18 2024-11-08T18:42:44,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/3bfc6eafee8e4eaf8af357f0e45a6ce2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/3bfc6eafee8e4eaf8af357f0e45a6ce2 2024-11-08T18:42:44,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/128565ca948b4969aa351d45e002af5f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/128565ca948b4969aa351d45e002af5f 2024-11-08T18:42:44,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/a34dd484f1874898b3fbc904ed05140e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/a34dd484f1874898b3fbc904ed05140e 2024-11-08T18:42:44,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/e9744cacfc324e4da5c1a861ec740c23 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/e9744cacfc324e4da5c1a861ec740c23 2024-11-08T18:42:44,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/442eff680547472d8ad78f72622469c2 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/442eff680547472d8ad78f72622469c2 2024-11-08T18:42:44,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/70f636107fdc412292ab9cbb4d0401b5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/70f636107fdc412292ab9cbb4d0401b5 2024-11-08T18:42:44,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/738de1da19fa49d0a2b9b8258db9f193 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/738de1da19fa49d0a2b9b8258db9f193 2024-11-08T18:42:44,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/18a1da38ef4041a98589bf2a2b667e3b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/18a1da38ef4041a98589bf2a2b667e3b 2024-11-08T18:42:44,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/4d55c21ca6e649319df244484b99514f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/4d55c21ca6e649319df244484b99514f 2024-11-08T18:42:44,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/b1406523d9a54fd4bd459c79595eb42a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/b1406523d9a54fd4bd459c79595eb42a 2024-11-08T18:42:44,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/94a51a413aac4a8198fba37de9a118b9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/94a51a413aac4a8198fba37de9a118b9 2024-11-08T18:42:44,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/fa82bead8e8e4bb898aa05cb31fb77cf to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/fa82bead8e8e4bb898aa05cb31fb77cf 2024-11-08T18:42:44,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5f17ec721eb64e4cb67720a57ac8bda6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/90b793e2d5e4452294b7ea45ecf16ade, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d762e6404b924407a2f7d79a51404366, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/076febbd578045cd8bb4fd7f223d39b9, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c001b158bc646aba88fcaa612d6053d, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3074005c29a94539a7c68b155e44e813, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/078c3e97ab3c42508982e3e7eb0300b4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c70aaa8118e44619e7ce9038a396ffb, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/4a9518ffdf4d46ab82997986dedaeefc, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/8995ae42bd73498fbc17da15e5d553db, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5333576dade64244951e665332e9b16f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d76ffee2fc1d4a31a004c5b963f4f6b4, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/62ff0269fc914b309c7d9ddd6cf6096e, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d4a4d6e95bbe41a783c27fa2d4874c54, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/13b7ad79c7024d90babde102ddc79268, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3d809364ad154b5eb77f66c47dc7186c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3f81075a36a845a1a305a24bdce07cb8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/89b67f9edf1b4d0984cb64e56feb237d] to archive 2024-11-08T18:42:44,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:42:44,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5f17ec721eb64e4cb67720a57ac8bda6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5f17ec721eb64e4cb67720a57ac8bda6 2024-11-08T18:42:44,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/90b793e2d5e4452294b7ea45ecf16ade to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/90b793e2d5e4452294b7ea45ecf16ade 2024-11-08T18:42:44,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d762e6404b924407a2f7d79a51404366 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d762e6404b924407a2f7d79a51404366 2024-11-08T18:42:44,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/076febbd578045cd8bb4fd7f223d39b9 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/076febbd578045cd8bb4fd7f223d39b9 2024-11-08T18:42:44,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c001b158bc646aba88fcaa612d6053d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c001b158bc646aba88fcaa612d6053d 2024-11-08T18:42:44,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3074005c29a94539a7c68b155e44e813 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3074005c29a94539a7c68b155e44e813 2024-11-08T18:42:44,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/078c3e97ab3c42508982e3e7eb0300b4 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/078c3e97ab3c42508982e3e7eb0300b4 2024-11-08T18:42:44,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c70aaa8118e44619e7ce9038a396ffb to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5c70aaa8118e44619e7ce9038a396ffb 2024-11-08T18:42:44,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/4a9518ffdf4d46ab82997986dedaeefc to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/4a9518ffdf4d46ab82997986dedaeefc 2024-11-08T18:42:44,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/8995ae42bd73498fbc17da15e5d553db to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/8995ae42bd73498fbc17da15e5d553db 2024-11-08T18:42:44,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5333576dade64244951e665332e9b16f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/5333576dade64244951e665332e9b16f 2024-11-08T18:42:44,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d76ffee2fc1d4a31a004c5b963f4f6b4 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d76ffee2fc1d4a31a004c5b963f4f6b4 2024-11-08T18:42:44,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/62ff0269fc914b309c7d9ddd6cf6096e to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/62ff0269fc914b309c7d9ddd6cf6096e 2024-11-08T18:42:44,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d4a4d6e95bbe41a783c27fa2d4874c54 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d4a4d6e95bbe41a783c27fa2d4874c54 2024-11-08T18:42:44,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/13b7ad79c7024d90babde102ddc79268 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/13b7ad79c7024d90babde102ddc79268 2024-11-08T18:42:44,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3d809364ad154b5eb77f66c47dc7186c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3d809364ad154b5eb77f66c47dc7186c 2024-11-08T18:42:44,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3f81075a36a845a1a305a24bdce07cb8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/3f81075a36a845a1a305a24bdce07cb8 2024-11-08T18:42:44,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/89b67f9edf1b4d0984cb64e56feb237d to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/89b67f9edf1b4d0984cb64e56feb237d 2024-11-08T18:42:44,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/13400cca08b44ae8963f0afd1e3f6e7f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f17b418d36924cb0ae797ad7aa4ab478, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5a9771f6ca6e4a25aa7cb7059d133d0c, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/b217e6b7e02840e5becd47e6d5a6b59a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c525900409244f5bb26f9485b09f9390, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8acfade59d0d4086876599b8f8df19d7, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/94fbed45b5ff4b8f8fff168ade50e193, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/3eb734bdf2e740769ed35d27cbca277a, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/020aa6b4204a4ccabac6facdd0aa5df8, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/cf98c8307b444bdfb80665b65f59aca6, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c622436585cb47139d3fafff59f89eea, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8b2a7c91ff7b4cc4b5cc3eeaaf17877f, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/7eef1a45f791416b83ab0664191f0567, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/81d2795fd7c7415292459881099a71e3, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/63491121dbf3460d80c64f9e022b4ef5, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5bccac82ef7249dfa76144cee6f570d1, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/a6ea6b9cb65b478ca19fd85b6c1d1066, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/64e2bfb7d3804328a3c9c55f810c3328] to archive 2024-11-08T18:42:44,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-08T18:42:44,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/13400cca08b44ae8963f0afd1e3f6e7f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/13400cca08b44ae8963f0afd1e3f6e7f 2024-11-08T18:42:44,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f17b418d36924cb0ae797ad7aa4ab478 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f17b418d36924cb0ae797ad7aa4ab478 2024-11-08T18:42:44,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5a9771f6ca6e4a25aa7cb7059d133d0c to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5a9771f6ca6e4a25aa7cb7059d133d0c 2024-11-08T18:42:44,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/b217e6b7e02840e5becd47e6d5a6b59a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/b217e6b7e02840e5becd47e6d5a6b59a 2024-11-08T18:42:44,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c525900409244f5bb26f9485b09f9390 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c525900409244f5bb26f9485b09f9390 2024-11-08T18:42:44,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8acfade59d0d4086876599b8f8df19d7 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8acfade59d0d4086876599b8f8df19d7 2024-11-08T18:42:44,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/94fbed45b5ff4b8f8fff168ade50e193 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/94fbed45b5ff4b8f8fff168ade50e193 2024-11-08T18:42:44,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/3eb734bdf2e740769ed35d27cbca277a to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/3eb734bdf2e740769ed35d27cbca277a 2024-11-08T18:42:44,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/020aa6b4204a4ccabac6facdd0aa5df8 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/020aa6b4204a4ccabac6facdd0aa5df8 2024-11-08T18:42:44,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/cf98c8307b444bdfb80665b65f59aca6 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/cf98c8307b444bdfb80665b65f59aca6 2024-11-08T18:42:44,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c622436585cb47139d3fafff59f89eea to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/c622436585cb47139d3fafff59f89eea 2024-11-08T18:42:44,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8b2a7c91ff7b4cc4b5cc3eeaaf17877f to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/8b2a7c91ff7b4cc4b5cc3eeaaf17877f 2024-11-08T18:42:44,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/7eef1a45f791416b83ab0664191f0567 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/7eef1a45f791416b83ab0664191f0567 2024-11-08T18:42:44,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/81d2795fd7c7415292459881099a71e3 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/81d2795fd7c7415292459881099a71e3 2024-11-08T18:42:44,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/63491121dbf3460d80c64f9e022b4ef5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/63491121dbf3460d80c64f9e022b4ef5 2024-11-08T18:42:44,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5bccac82ef7249dfa76144cee6f570d1 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/5bccac82ef7249dfa76144cee6f570d1 2024-11-08T18:42:44,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/a6ea6b9cb65b478ca19fd85b6c1d1066 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/a6ea6b9cb65b478ca19fd85b6c1d1066 2024-11-08T18:42:44,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/64e2bfb7d3804328a3c9c55f810c3328 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/64e2bfb7d3804328a3c9c55f810c3328 2024-11-08T18:42:44,419 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/recovered.edits/288.seqid, newMaxSeqId=288, maxSeqId=4 2024-11-08T18:42:44,419 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. 2024-11-08T18:42:44,419 DEBUG [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] regionserver.HRegion(1676): Region close journal for e98ed84368420c18855ecb5c81bb1538: Waiting for close lock at 1731091362616Running coprocessor pre-close hooks at 1731091362616Disabling compacts and flushes for region at 1731091362616Disabling writes for close at 1731091362617 (+1 ms)Obtaining lock to block concurrent updates at 1731091362617Preparing flush snapshotting stores in e98ed84368420c18855ecb5c81bb1538 at 1731091362617Finished memstore snapshotting TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731091362618 (+1 ms)Flushing stores of TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538. at 1731091362619 (+1 ms)Flushing e98ed84368420c18855ecb5c81bb1538/A: creating writer at 1731091362619Flushing mob file e98ed84368420c18855ecb5c81bb1538/A: appending metadata at 1731091362627 (+8 ms)Flushing mob file e98ed84368420c18855ecb5c81bb1538/A: closing flushed file at 1731091362627Flushing e98ed84368420c18855ecb5c81bb1538/A: appending metadata at 1731091363042 (+415 ms)Flushing e98ed84368420c18855ecb5c81bb1538/A: closing flushed file at 1731091363043 (+1 ms)Flushing e98ed84368420c18855ecb5c81bb1538/B: creating writer at 1731091363459 (+416 ms)Flushing e98ed84368420c18855ecb5c81bb1538/B: appending metadata at 1731091363463 (+4 ms)Flushing e98ed84368420c18855ecb5c81bb1538/B: closing flushed file at 1731091363463Flushing e98ed84368420c18855ecb5c81bb1538/C: creating writer at 1731091363876 (+413 ms)Flushing e98ed84368420c18855ecb5c81bb1538/C: appending metadata at 1731091363881 (+5 ms)Flushing e98ed84368420c18855ecb5c81bb1538/C: closing flushed file at 1731091363881Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@32de41d1: reopening flushed file at 1731091364329 (+448 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a8351bf: reopening flushed file at 1731091364334 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@581dffe0: reopening flushed file at 1731091364339 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for e98ed84368420c18855ecb5c81bb1538 in 1726ms, sequenceid=285, compaction requested=true at 1731091364344 (+5 ms)Writing region close event to WAL at 1731091364417 (+73 ms)Running coprocessor post-close hooks at 1731091364419 (+2 ms)Closed at 1731091364419 2024-11-08T18:42:44,421 INFO [RS_CLOSE_REGION-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_REGION, pid=161}] handler.UnassignRegionHandler(157): Closed e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,421 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=160 updating hbase:meta row=e98ed84368420c18855ecb5c81bb1538, regionState=CLOSED 2024-11-08T18:42:44,422 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=160, state=RUNNABLE, hasLock=false; CloseRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 because future has completed 2024-11-08T18:42:44,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-08T18:42:44,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; CloseRegionProcedure e98ed84368420c18855ecb5c81bb1538, server=b75fe890c219,44447,1731091183487 in 1.9610 sec 2024-11-08T18:42:44,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-08T18:42:44,426 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e98ed84368420c18855ecb5c81bb1538, UNASSIGN in 1.9660 sec 2024-11-08T18:42:44,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-08T18:42:44,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9700 sec 2024-11-08T18:42:44,428 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731091364428"}]},"ts":"1731091364428"} 2024-11-08T18:42:44,429 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-08T18:42:44,429 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-08T18:42:44,430 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9850 sec 2024-11-08T18:42:44,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-08T18:42:44,600 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:44,600 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-08T18:42:44,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:44,603 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=162, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:44,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T18:42:44,605 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=162, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:44,609 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,614 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C, FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/recovered.edits] 2024-11-08T18:42:44,619 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/5c2132cd602c4554b213268225ff9849 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/5c2132cd602c4554b213268225ff9849 2024-11-08T18:42:44,621 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/882d83836ddc4667ab34007e28719c0b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/882d83836ddc4667ab34007e28719c0b 2024-11-08T18:42:44,623 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f1482f8a217d49818c773096430a8b5b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/A/f1482f8a217d49818c773096430a8b5b 2024-11-08T18:42:44,627 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/13a6f878d1734ba0a993c937cf304e12 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/13a6f878d1734ba0a993c937cf304e12 2024-11-08T18:42:44,628 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/74aeaaaf23c841d8991aee9523a34a48 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/74aeaaaf23c841d8991aee9523a34a48 2024-11-08T18:42:44,628 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d578b2f2824f4210a79b6c6cb01f47a5 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/B/d578b2f2824f4210a79b6c6cb01f47a5 2024-11-08T18:42:44,630 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/4489934285fd49f69df03f215863583b to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/4489934285fd49f69df03f215863583b 2024-11-08T18:42:44,630 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/e56a2b43f05f47288bf8336b131db938 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/e56a2b43f05f47288bf8336b131db938 2024-11-08T18:42:44,631 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f1ba81ea61bf4f7a93d2b27765bd2d62 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/C/f1ba81ea61bf4f7a93d2b27765bd2d62 2024-11-08T18:42:44,633 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/recovered.edits/288.seqid to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538/recovered.edits/288.seqid 2024-11-08T18:42:44,633 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/default/TestAcidGuarantees/e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,633 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-08T18:42:44,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T18:42:44,634 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-08T18:42:44,636 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081650f9f8696340459d92e9aebc92f68b_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081650f9f8696340459d92e9aebc92f68b_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,637 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110817b408f3338048e28442d771cfd7b9db_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110817b408f3338048e28442d771cfd7b9db_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081d0345b1e26249aaa67d960b599eec24_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411081d0345b1e26249aaa67d960b599eec24_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,638 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082253b64b7de9408fb6ebe0a8f509f730_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082253b64b7de9408fb6ebe0a8f509f730_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,639 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110823a76e8cde694777bd09ac11aa849e21_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110823a76e8cde694777bd09ac11aa849e21_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110829efab9a189b44d481b8c893c419ecd0_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110829efab9a189b44d481b8c893c419ecd0_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,640 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082efb6b9962c24d888254df6d13bb5758_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411082efb6b9962c24d888254df6d13bb5758_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,641 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845a5fa70366447a4a6698a7b5cd7ea52_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110845a5fa70366447a4a6698a7b5cd7ea52_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110853b3eb5b9cf544ac8358ee83a1b1502e_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110853b3eb5b9cf544ac8358ee83a1b1502e_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,642 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b4edc2e61fe48e9a2e8eaa0cb62fe66_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411085b4edc2e61fe48e9a2e8eaa0cb62fe66_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,643 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108786ff26246eb4d60bbdef5430729f2bc_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108786ff26246eb4d60bbdef5430729f2bc_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110892adfab4fcaa4fefb15bbdc84c04f0b1_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110892adfab4fcaa4fefb15bbdc84c04f0b1_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,644 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a758e2a67c5e43f8aaa8c8e0d8eb9d48_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108a758e2a67c5e43f8aaa8c8e0d8eb9d48_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c32d0adb1b5e41d9ac4804e8851129b3_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108c32d0adb1b5e41d9ac4804e8851129b3_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,645 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cdfa3e7d437842f1927fee0bea8d5420_e98ed84368420c18855ecb5c81bb1538 to hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241108cdfa3e7d437842f1927fee0bea8d5420_e98ed84368420c18855ecb5c81bb1538 2024-11-08T18:42:44,646 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-08T18:42:44,647 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=162, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:44,650 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-08T18:42:44,652 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-08T18:42:44,653 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=162, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:44,653 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-08T18:42:44,653 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731091364653"}]},"ts":"9223372036854775807"} 2024-11-08T18:42:44,655 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-08T18:42:44,655 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e98ed84368420c18855ecb5c81bb1538, NAME => 'TestAcidGuarantees,,1731091336144.e98ed84368420c18855ecb5c81bb1538.', STARTKEY => '', ENDKEY => ''}] 2024-11-08T18:42:44,655 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-08T18:42:44,655 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731091364655"}]},"ts":"9223372036854775807"} 2024-11-08T18:42:44,657 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-08T18:42:44,657 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=162, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-08T18:42:44,658 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 57 msec 2024-11-08T18:42:44,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42293 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-08T18:42:44,709 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-08T18:42:44,709 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-08T18:42:44,716 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=240 (was 238) - Thread LEAK? -, OpenFileDescriptor=451 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=301 (was 307), ProcessCount=11 (was 11), AvailableMemoryMB=3681 (was 3715) 2024-11-08T18:42:44,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-08T18:42:44,716 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T18:42:44,716 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:42:44,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:44,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:44,716 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-08T18:42:44,716 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-08T18:42:44,716 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=855483586, stopped=false 2024-11-08T18:42:44,717 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=b75fe890c219,42293,1731091182659 2024-11-08T18:42:44,774 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T18:42:44,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-08T18:42:44,774 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:42:44,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:42:44,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T18:42:44,775 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-08T18:42:44,775 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:42:44,775 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:44,776 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T18:42:44,776 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'b75fe890c219,44447,1731091183487' ***** 2024-11-08T18:42:44,776 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-08T18:42:44,777 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-08T18:42:44,777 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-08T18:42:44,778 INFO [RS:0;b75fe890c219:44447 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-08T18:42:44,778 INFO [RS:0;b75fe890c219:44447 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-08T18:42:44,778 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(959): stopping server b75fe890c219,44447,1731091183487 2024-11-08T18:42:44,778 INFO [RS:0;b75fe890c219:44447 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T18:42:44,778 INFO [RS:0;b75fe890c219:44447 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;b75fe890c219:44447. 2024-11-08T18:42:44,778 DEBUG [RS:0;b75fe890c219:44447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-08T18:42:44,778 DEBUG [RS:0;b75fe890c219:44447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:44,779 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-08T18:42:44,779 INFO [RS:0;b75fe890c219:44447 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-08T18:42:44,779 INFO [RS:0;b75fe890c219:44447 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-08T18:42:44,779 INFO [RS:0;b75fe890c219:44447 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-08T18:42:44,779 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-08T18:42:44,780 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-08T18:42:44,780 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-08T18:42:44,780 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:44,780 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-08T18:42:44,780 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-08T18:42:44,780 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-08T18:42:44,780 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-08T18:42:44,780 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-08T18:42:44,781 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-08T18:42:44,807 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/info/c8dbdfc9f85b4fe1a7b94b61a388d45a is 95, key is TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282./info:/1731091218573/DeleteFamily/seqid=0 2024-11-08T18:42:44,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742414_1590 (size=6514) 2024-11-08T18:42:44,863 INFO [regionserver/b75fe890c219:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T18:42:44,981 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:45,129 INFO [regionserver/b75fe890c219:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-08T18:42:45,129 INFO [regionserver/b75fe890c219:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-08T18:42:45,181 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:45,212 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/info/c8dbdfc9f85b4fe1a7b94b61a388d45a 2024-11-08T18:42:45,236 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/ns/a3d7efc3e55c4a349cee0662d4ffccd7 is 93, key is TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282./ns:/1731091218523/DeleteFamily/seqid=0 2024-11-08T18:42:45,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742415_1591 (size=5906) 2024-11-08T18:42:45,381 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:45,581 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:45,640 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/ns/a3d7efc3e55c4a349cee0662d4ffccd7 2024-11-08T18:42:45,666 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/rep_barrier/48d5c44a20bd48d28804bcb691fc212d is 102, key is TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282./rep_barrier:/1731091218523/DeleteFamily/seqid=0 2024-11-08T18:42:45,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742416_1592 (size=6025) 2024-11-08T18:42:45,782 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:45,982 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:46,070 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/rep_barrier/48d5c44a20bd48d28804bcb691fc212d 2024-11-08T18:42:46,097 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/table/89beaeb2edd24ecfa907d752236edbe7 is 96, key is TestAcidGuarantees,,1731091187217.b4a84f4a6ffc3bb08667e2c0f91ea282./table:/1731091218523/DeleteFamily/seqid=0 2024-11-08T18:42:46,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742417_1593 (size=5936) 2024-11-08T18:42:46,182 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:46,195 INFO [regionserver/b75fe890c219:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-08T18:42:46,382 DEBUG [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-08T18:42:46,502 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/table/89beaeb2edd24ecfa907d752236edbe7 2024-11-08T18:42:46,512 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/info/c8dbdfc9f85b4fe1a7b94b61a388d45a as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/info/c8dbdfc9f85b4fe1a7b94b61a388d45a 2024-11-08T18:42:46,520 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/info/c8dbdfc9f85b4fe1a7b94b61a388d45a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-08T18:42:46,522 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/ns/a3d7efc3e55c4a349cee0662d4ffccd7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/ns/a3d7efc3e55c4a349cee0662d4ffccd7 2024-11-08T18:42:46,527 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/ns/a3d7efc3e55c4a349cee0662d4ffccd7, entries=8, sequenceid=90, filesize=5.8 K 2024-11-08T18:42:46,527 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/rep_barrier/48d5c44a20bd48d28804bcb691fc212d as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/rep_barrier/48d5c44a20bd48d28804bcb691fc212d 2024-11-08T18:42:46,531 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/rep_barrier/48d5c44a20bd48d28804bcb691fc212d, entries=6, sequenceid=90, filesize=5.9 K 2024-11-08T18:42:46,532 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/.tmp/table/89beaeb2edd24ecfa907d752236edbe7 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/table/89beaeb2edd24ecfa907d752236edbe7 2024-11-08T18:42:46,535 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/table/89beaeb2edd24ecfa907d752236edbe7, entries=7, sequenceid=90, filesize=5.8 K 2024-11-08T18:42:46,536 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1755ms, sequenceid=90, compaction requested=false 2024-11-08T18:42:46,542 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-08T18:42:46,542 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-08T18:42:46,543 INFO [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-08T18:42:46,543 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731091364780Running coprocessor pre-close hooks at 1731091364780Disabling compacts and flushes for region at 1731091364780Disabling writes for close at 1731091364780Obtaining lock to block concurrent updates at 1731091364781 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731091364781Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731091364781Flushing stores of hbase:meta,,1.1588230740 at 1731091364782 (+1 ms)Flushing 1588230740/info: creating writer at 1731091364782Flushing 1588230740/info: appending metadata at 1731091364807 (+25 ms)Flushing 1588230740/info: closing flushed file at 1731091364807Flushing 1588230740/ns: creating writer at 1731091365221 (+414 ms)Flushing 1588230740/ns: appending metadata at 1731091365235 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731091365236 (+1 ms)Flushing 1588230740/rep_barrier: creating writer at 1731091365646 (+410 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731091365666 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731091365666Flushing 1588230740/table: creating writer at 1731091366079 (+413 ms)Flushing 1588230740/table: appending metadata at 1731091366097 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731091366097Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5fe0e9af: reopening flushed file at 1731091366511 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6722ab6: reopening flushed file at 1731091366521 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@637e8eb2: reopening flushed file at 1731091366527 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1b47a878: reopening flushed file at 1731091366531 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1755ms, sequenceid=90, compaction requested=false at 1731091366536 (+5 ms)Writing region close event to WAL at 1731091366538 (+2 ms)Running coprocessor post-close hooks at 1731091366542 (+4 ms)Closed at 1731091366543 (+1 ms) 2024-11-08T18:42:46,543 DEBUG [RS_CLOSE_META-regionserver/b75fe890c219:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-08T18:42:46,583 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(976): stopping server b75fe890c219,44447,1731091183487; all regions closed. 2024-11-08T18:42:46,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741834_1010 (size=25344) 2024-11-08T18:42:46,596 DEBUG [RS:0;b75fe890c219:44447 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/oldWALs 2024-11-08T18:42:46,596 INFO [RS:0;b75fe890c219:44447 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b75fe890c219%2C44447%2C1731091183487.meta:.meta(num 1731091186664) 2024-11-08T18:42:46,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741833_1009 (size=13166967) 2024-11-08T18:42:46,602 DEBUG [RS:0;b75fe890c219:44447 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/oldWALs 2024-11-08T18:42:46,602 INFO [RS:0;b75fe890c219:44447 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL b75fe890c219%2C44447%2C1731091183487:(num 1731091186284) 2024-11-08T18:42:46,602 DEBUG [RS:0;b75fe890c219:44447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-08T18:42:46,602 INFO [RS:0;b75fe890c219:44447 {}] regionserver.LeaseManager(133): Closed leases 2024-11-08T18:42:46,602 INFO [RS:0;b75fe890c219:44447 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T18:42:46,602 INFO [RS:0;b75fe890c219:44447 {}] hbase.ChoreService(370): Chore service for: regionserver/b75fe890c219:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-08T18:42:46,602 INFO [RS:0;b75fe890c219:44447 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T18:42:46,603 INFO [regionserver/b75fe890c219:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T18:42:46,603 INFO [RS:0;b75fe890c219:44447 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44447 2024-11-08T18:42:46,665 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/b75fe890c219,44447,1731091183487 2024-11-08T18:42:46,665 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-08T18:42:46,665 INFO [RS:0;b75fe890c219:44447 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T18:42:46,673 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [b75fe890c219,44447,1731091183487] 2024-11-08T18:42:46,682 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/b75fe890c219,44447,1731091183487 already deleted, retry=false 2024-11-08T18:42:46,682 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; b75fe890c219,44447,1731091183487 expired; onlineServers=0 2024-11-08T18:42:46,682 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'b75fe890c219,42293,1731091182659' ***** 2024-11-08T18:42:46,682 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-08T18:42:46,682 INFO [M:0;b75fe890c219:42293 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-08T18:42:46,683 INFO [M:0;b75fe890c219:42293 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-08T18:42:46,683 DEBUG [M:0;b75fe890c219:42293 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-08T18:42:46,683 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-08T18:42:46,683 DEBUG [M:0;b75fe890c219:42293 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-08T18:42:46,683 DEBUG [master/b75fe890c219:0:becomeActiveMaster-HFileCleaner.large.0-1731091185932 {}] cleaner.HFileCleaner(306): Exit Thread[master/b75fe890c219:0:becomeActiveMaster-HFileCleaner.large.0-1731091185932,5,FailOnTimeoutGroup] 2024-11-08T18:42:46,683 DEBUG [master/b75fe890c219:0:becomeActiveMaster-HFileCleaner.small.0-1731091185933 {}] cleaner.HFileCleaner(306): Exit Thread[master/b75fe890c219:0:becomeActiveMaster-HFileCleaner.small.0-1731091185933,5,FailOnTimeoutGroup] 2024-11-08T18:42:46,684 INFO [M:0;b75fe890c219:42293 {}] hbase.ChoreService(370): Chore service for: master/b75fe890c219:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-08T18:42:46,684 INFO [M:0;b75fe890c219:42293 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-08T18:42:46,684 DEBUG [M:0;b75fe890c219:42293 {}] master.HMaster(1795): Stopping service threads 2024-11-08T18:42:46,684 INFO [M:0;b75fe890c219:42293 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-08T18:42:46,684 INFO [M:0;b75fe890c219:42293 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-08T18:42:46,685 ERROR [M:0;b75fe890c219:42293 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-08T18:42:46,686 INFO [M:0;b75fe890c219:42293 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-08T18:42:46,686 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-08T18:42:46,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-08T18:42:46,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-08T18:42:46,690 DEBUG [M:0;b75fe890c219:42293 {}] zookeeper.ZKUtil(347): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-08T18:42:46,690 WARN [M:0;b75fe890c219:42293 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-08T18:42:46,691 INFO [M:0;b75fe890c219:42293 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/.lastflushedseqids 2024-11-08T18:42:46,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742418_1594 (size=72) 2024-11-08T18:42:46,774 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T18:42:46,774 DEBUG [pool-37-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44447-0x1011d08c3510001, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T18:42:46,775 INFO [RS:0;b75fe890c219:44447 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T18:42:46,776 INFO [RS:0;b75fe890c219:44447 {}] regionserver.HRegionServer(1031): Exiting; stopping=b75fe890c219,44447,1731091183487; zookeeper connection closed. 2024-11-08T18:42:46,776 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@30aa0683 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@30aa0683 2024-11-08T18:42:46,777 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-08T18:42:47,103 INFO [M:0;b75fe890c219:42293 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-08T18:42:47,104 INFO [M:0;b75fe890c219:42293 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-08T18:42:47,104 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-08T18:42:47,104 INFO [M:0;b75fe890c219:42293 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T18:42:47,104 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T18:42:47,105 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-08T18:42:47,105 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T18:42:47,105 INFO [M:0;b75fe890c219:42293 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=734.15 KB heapSize=900.36 KB 2024-11-08T18:42:47,128 DEBUG [M:0;b75fe890c219:42293 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/19e607ab6a774c43a57c3cef6ddbce63 is 82, key is hbase:meta,,1/info:regioninfo/1731091186786/Put/seqid=0 2024-11-08T18:42:47,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742419_1595 (size=5672) 2024-11-08T18:42:47,533 INFO [M:0;b75fe890c219:42293 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2048 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/19e607ab6a774c43a57c3cef6ddbce63 2024-11-08T18:42:47,563 DEBUG [M:0;b75fe890c219:42293 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbb9dffd0da242ceb1560dbee163b223 is 2276, key is \x00\x00\x00\x00\x00\x00\x00Z/proc:d/1731091278006/Put/seqid=0 2024-11-08T18:42:47,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742420_1596 (size=42808) 2024-11-08T18:42:47,968 INFO [M:0;b75fe890c219:42293 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=733.59 KB at sequenceid=2048 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbb9dffd0da242ceb1560dbee163b223 2024-11-08T18:42:47,977 INFO [M:0;b75fe890c219:42293 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fbb9dffd0da242ceb1560dbee163b223 2024-11-08T18:42:47,994 DEBUG [M:0;b75fe890c219:42293 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67ebafcc610449df81c7d18937c18af3 is 69, key is b75fe890c219,44447,1731091183487/rs:state/1731091186044/Put/seqid=0 2024-11-08T18:42:47,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073742421_1597 (size=5156) 2024-11-08T18:42:48,399 INFO [M:0;b75fe890c219:42293 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2048 (bloomFilter=true), to=hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67ebafcc610449df81c7d18937c18af3 2024-11-08T18:42:48,409 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/19e607ab6a774c43a57c3cef6ddbce63 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/19e607ab6a774c43a57c3cef6ddbce63 2024-11-08T18:42:48,412 INFO [M:0;b75fe890c219:42293 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/19e607ab6a774c43a57c3cef6ddbce63, entries=8, sequenceid=2048, filesize=5.5 K 2024-11-08T18:42:48,413 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fbb9dffd0da242ceb1560dbee163b223 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbb9dffd0da242ceb1560dbee163b223 2024-11-08T18:42:48,417 INFO [M:0;b75fe890c219:42293 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fbb9dffd0da242ceb1560dbee163b223 2024-11-08T18:42:48,417 INFO [M:0;b75fe890c219:42293 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fbb9dffd0da242ceb1560dbee163b223, entries=162, sequenceid=2048, filesize=41.8 K 2024-11-08T18:42:48,417 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/67ebafcc610449df81c7d18937c18af3 as hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67ebafcc610449df81c7d18937c18af3 2024-11-08T18:42:48,421 INFO [M:0;b75fe890c219:42293 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40013/user/jenkins/test-data/b1d14014-3557-29b4-866f-3f415532c787/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/67ebafcc610449df81c7d18937c18af3, entries=1, sequenceid=2048, filesize=5.0 K 2024-11-08T18:42:48,422 INFO [M:0;b75fe890c219:42293 {}] regionserver.HRegion(3140): Finished flush of dataSize ~734.15 KB/751768, heapSize ~900.06 KB/921664, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1317ms, sequenceid=2048, compaction requested=false 2024-11-08T18:42:48,423 INFO [M:0;b75fe890c219:42293 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-08T18:42:48,423 DEBUG [M:0;b75fe890c219:42293 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731091367104Disabling compacts and flushes for region at 1731091367104Disabling writes for close at 1731091367105 (+1 ms)Obtaining lock to block concurrent updates at 1731091367105Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731091367105Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=751768, getHeapSize=921904, getOffHeapSize=0, getCellsCount=2235 at 1731091367106 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731091367107 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731091367107Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731091367128 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731091367128Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731091367543 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731091367562 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731091367562Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731091367977 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731091367994 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731091367994Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20fe096f: reopening flushed file at 1731091368408 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b5eed20: reopening flushed file at 1731091368412 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@316fc074: reopening flushed file at 1731091368417 (+5 ms)Finished flush of dataSize ~734.15 KB/751768, heapSize ~900.06 KB/921664, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1317ms, sequenceid=2048, compaction requested=false at 1731091368422 (+5 ms)Writing region close event to WAL at 1731091368423 (+1 ms)Closed at 1731091368423 2024-11-08T18:42:48,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46719 is added to blk_1073741830_1006 (size=886055) 2024-11-08T18:42:48,429 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-08T18:42:48,429 INFO [M:0;b75fe890c219:42293 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-08T18:42:48,429 INFO [M:0;b75fe890c219:42293 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42293 2024-11-08T18:42:48,429 INFO [M:0;b75fe890c219:42293 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-08T18:42:48,590 INFO [M:0;b75fe890c219:42293 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-08T18:42:48,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T18:42:48,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42293-0x1011d08c3510000, quorum=127.0.0.1:49215, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-08T18:42:48,597 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-08T18:42:48,601 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T18:42:48,601 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T18:42:48,602 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T18:42:48,602 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/hadoop.log.dir/,STOPPED} 2024-11-08T18:42:48,605 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-08T18:42:48,605 WARN [BP-656518607-172.17.0.2-1731091178482 heartbeating to localhost/127.0.0.1:40013 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-08T18:42:48,605 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-08T18:42:48,605 WARN [BP-656518607-172.17.0.2-1731091178482 heartbeating to localhost/127.0.0.1:40013 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-656518607-172.17.0.2-1731091178482 (Datanode Uuid d9e2d767-d14a-4c65-9b4d-aa0a9bb7ed83) service to localhost/127.0.0.1:40013 2024-11-08T18:42:48,607 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/data/data1/current/BP-656518607-172.17.0.2-1731091178482 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T18:42:48,608 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/cluster_9e57a258-9f46-7d18-ec73-9720402f917a/data/data2/current/BP-656518607-172.17.0.2-1731091178482 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-08T18:42:48,608 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-08T18:42:48,615 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-08T18:42:48,616 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-08T18:42:48,616 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-08T18:42:48,616 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-08T18:42:48,616 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3702ee0b-97de-5074-bb79-be6d626189cc/hadoop.log.dir/,STOPPED} 2024-11-08T18:42:48,629 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-08T18:42:48,736 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down